diff options
author | Chocobozzz <me@florianbigard.com> | 2023-04-21 14:55:10 +0200 |
---|---|---|
committer | Chocobozzz <chocobozzz@cpy.re> | 2023-05-09 08:57:34 +0200 |
commit | 0c9668f77901e7540e2c7045eb0f2974a4842a69 (patch) | |
tree | 226d3dd1565b0bb56588897af3b8530e6216e96b | |
parent | 6bcb854cdea8688a32240bc5719c7d139806e00b (diff) | |
download | PeerTube-0c9668f77901e7540e2c7045eb0f2974a4842a69.tar.gz PeerTube-0c9668f77901e7540e2c7045eb0f2974a4842a69.tar.zst PeerTube-0c9668f77901e7540e2c7045eb0f2974a4842a69.zip |
Implement remote runner jobs in server
Move ffmpeg functions to @shared
168 files changed, 6906 insertions, 2802 deletions
diff --git a/config/default.yaml b/config/default.yaml index dfa43a0aa..986b2e999 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -375,6 +375,12 @@ feeds: | |||
375 | # Default number of comments displayed in feeds | 375 | # Default number of comments displayed in feeds |
376 | count: 20 | 376 | count: 20 |
377 | 377 | ||
378 | remote_runners: | ||
379 | # Consider jobs that are processed by a remote runner as stalled after this period of time without any update | ||
380 | stalled_jobs: | ||
381 | live: '30 seconds' | ||
382 | vod: '2 minutes' | ||
383 | |||
378 | cache: | 384 | cache: |
379 | previews: | 385 | previews: |
380 | size: 500 # Max number of previews you want to cache | 386 | size: 500 # Max number of previews you want to cache |
@@ -433,12 +439,18 @@ transcoding: | |||
433 | # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file | 439 | # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file |
434 | allow_audio_files: true | 440 | allow_audio_files: true |
435 | 441 | ||
436 | # Amount of threads used by ffmpeg for 1 transcoding job | 442 | # Enable remote runners to transcode your videos |
443 | # If enabled, your instance won't transcode the videos itself | ||
444 | # At least 1 remote runner must be configured to transcode your videos | ||
445 | remote_runners: | ||
446 | enabled: false | ||
447 | |||
448 | # Amount of threads used by ffmpeg for 1 local transcoding job | ||
437 | threads: 1 | 449 | threads: 1 |
438 | # Amount of transcoding jobs to execute in parallel | 450 | # Amount of local transcoding jobs to execute in parallel |
439 | concurrency: 1 | 451 | concurrency: 1 |
440 | 452 | ||
441 | # Choose the transcoding profile | 453 | # Choose the local transcoding profile |
442 | # New profiles can be added by plugins | 454 | # New profiles can be added by plugins |
443 | # Available in core PeerTube: 'default' | 455 | # Available in core PeerTube: 'default' |
444 | profile: 'default' | 456 | profile: 'default' |
@@ -533,9 +545,17 @@ live: | |||
533 | # Allow to transcode the live streaming in multiple live resolutions | 545 | # Allow to transcode the live streaming in multiple live resolutions |
534 | transcoding: | 546 | transcoding: |
535 | enabled: true | 547 | enabled: true |
548 | |||
549 | # Enable remote runners to transcode your videos | ||
550 | # If enabled, your instance won't transcode the videos itself | ||
551 | # At least 1 remote runner must be configured to transcode your videos | ||
552 | remote_runners: | ||
553 | enabled: false | ||
554 | |||
555 | # Amount of threads used by ffmpeg per live when using local transcoding | ||
536 | threads: 2 | 556 | threads: 2 |
537 | 557 | ||
538 | # Choose the transcoding profile | 558 | # Choose the local transcoding profile |
539 | # New profiles can be added by plugins | 559 | # New profiles can be added by plugins |
540 | # Available in core PeerTube: 'default' | 560 | # Available in core PeerTube: 'default' |
541 | profile: 'default' | 561 | profile: 'default' |
@@ -754,7 +774,7 @@ search: | |||
754 | search_index: | 774 | search_index: |
755 | enabled: false | 775 | enabled: false |
756 | # URL of the search index, that should use the same search API and routes | 776 | # URL of the search index, that should use the same search API and routes |
757 | # than PeerTube: https://docs.joinpeertube.org/api/rest-reference.html | 777 | # than PeerTube: https://docs.joinpeertube.org/api-rest-reference.html |
758 | # You should deploy your own with https://framagit.org/framasoft/peertube/search-index, | 778 | # You should deploy your own with https://framagit.org/framasoft/peertube/search-index, |
759 | # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index | 779 | # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index |
760 | url: '' | 780 | url: '' |
diff --git a/config/production.yaml.example b/config/production.yaml.example index 0fb6ababc..bd01375cd 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example | |||
@@ -373,6 +373,12 @@ feeds: | |||
373 | # Default number of comments displayed in feeds | 373 | # Default number of comments displayed in feeds |
374 | count: 20 | 374 | count: 20 |
375 | 375 | ||
376 | remote_runners: | ||
377 | # Consider jobs that are processed by a remote runner as stalled after this period of time without any update | ||
378 | stalled_jobs: | ||
379 | live: '30 seconds' | ||
380 | vod: '2 minutes' | ||
381 | |||
376 | ############################################################################### | 382 | ############################################################################### |
377 | # | 383 | # |
378 | # From this point, almost all following keys can be overridden by the web interface | 384 | # From this point, almost all following keys can be overridden by the web interface |
@@ -443,12 +449,18 @@ transcoding: | |||
443 | # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file | 449 | # If a user uploads an audio file, PeerTube will create a video by merging the preview file and the audio file |
444 | allow_audio_files: true | 450 | allow_audio_files: true |
445 | 451 | ||
446 | # Amount of threads used by ffmpeg for 1 transcoding job | 452 | # Enable remote runners to transcode your videos |
453 | # If enabled, your instance won't transcode the videos itself | ||
454 | # At least 1 remote runner must be configured to transcode your videos | ||
455 | remote_runners: | ||
456 | enabled: false | ||
457 | |||
458 | # Amount of threads used by ffmpeg for 1 local transcoding job | ||
447 | threads: 1 | 459 | threads: 1 |
448 | # Amount of transcoding jobs to execute in parallel | 460 | # Amount of local transcoding jobs to execute in parallel |
449 | concurrency: 1 | 461 | concurrency: 1 |
450 | 462 | ||
451 | # Choose the transcoding profile | 463 | # Choose the local transcoding profile |
452 | # New profiles can be added by plugins | 464 | # New profiles can be added by plugins |
453 | # Available in core PeerTube: 'default' | 465 | # Available in core PeerTube: 'default' |
454 | profile: 'default' | 466 | profile: 'default' |
@@ -543,9 +555,17 @@ live: | |||
543 | # Allow to transcode the live streaming in multiple live resolutions | 555 | # Allow to transcode the live streaming in multiple live resolutions |
544 | transcoding: | 556 | transcoding: |
545 | enabled: true | 557 | enabled: true |
558 | |||
559 | # Enable remote runners to transcode your videos | ||
560 | # If enabled, your instance won't transcode the videos itself | ||
561 | # At least 1 remote runner must be configured to transcode your videos | ||
562 | remote_runners: | ||
563 | enabled: false | ||
564 | |||
565 | # Amount of threads used by ffmpeg per live when using local transcoding | ||
546 | threads: 2 | 566 | threads: 2 |
547 | 567 | ||
548 | # Choose the transcoding profile | 568 | # Choose the local transcoding profile |
549 | # New profiles can be added by plugins | 569 | # New profiles can be added by plugins |
550 | # Available in core PeerTube: 'default' | 570 | # Available in core PeerTube: 'default' |
551 | profile: 'default' | 571 | profile: 'default' |
@@ -607,7 +627,7 @@ import: | |||
607 | # See https://docs.joinpeertube.org/maintain/configuration#security for more information | 627 | # See https://docs.joinpeertube.org/maintain/configuration#security for more information |
608 | enabled: false | 628 | enabled: false |
609 | 629 | ||
610 | # Add ability for your users to synchronize their channels with external channels, playlists, etc. | 630 | # Add ability for your users to synchronize their channels with external channels, playlists, etc |
611 | video_channel_synchronization: | 631 | video_channel_synchronization: |
612 | enabled: false | 632 | enabled: false |
613 | 633 | ||
@@ -768,9 +788,9 @@ search: | |||
768 | # You should deploy your own with https://framagit.org/framasoft/peertube/search-index, | 788 | # You should deploy your own with https://framagit.org/framasoft/peertube/search-index, |
769 | # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index | 789 | # and can use https://search.joinpeertube.org/ for tests, but keep in mind the latter is an unmoderated search index |
770 | url: '' | 790 | url: '' |
771 | # You can disable local search, so users only use the search index | 791 | # You can disable local search in the client, so users only use the search index |
772 | disable_local_search: false | 792 | disable_local_search: false |
773 | # If you did not disable local search, you can decide to use the search index by default | 793 | # If you did not disable local search in the client, you can decide to use the search index by default |
774 | is_default_search: false | 794 | is_default_search: false |
775 | 795 | ||
776 | # PeerTube client/interface configuration | 796 | # PeerTube client/interface configuration |
@@ -133,6 +133,7 @@ import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-in | |||
133 | import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' | 133 | import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' |
134 | import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler' | 134 | import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler' |
135 | import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler' | 135 | import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler' |
136 | import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler' | ||
136 | import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' | 137 | import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' |
137 | import { PeerTubeSocket } from './server/lib/peertube-socket' | 138 | import { PeerTubeSocket } from './server/lib/peertube-socket' |
138 | import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' | 139 | import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' |
@@ -331,6 +332,7 @@ async function startApplication () { | |||
331 | VideoChannelSyncLatestScheduler.Instance.enable() | 332 | VideoChannelSyncLatestScheduler.Instance.enable() |
332 | VideoViewsBufferScheduler.Instance.enable() | 333 | VideoViewsBufferScheduler.Instance.enable() |
333 | GeoIPUpdateScheduler.Instance.enable() | 334 | GeoIPUpdateScheduler.Instance.enable() |
335 | RunnerJobWatchDogScheduler.Instance.enable() | ||
334 | 336 | ||
335 | OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) | 337 | OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) |
336 | 338 | ||
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 60d168d12..0b9aaffda 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -217,6 +217,9 @@ function customConfig (): CustomConfig { | |||
217 | }, | 217 | }, |
218 | transcoding: { | 218 | transcoding: { |
219 | enabled: CONFIG.TRANSCODING.ENABLED, | 219 | enabled: CONFIG.TRANSCODING.ENABLED, |
220 | remoteRunners: { | ||
221 | enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
222 | }, | ||
220 | allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS, | 223 | allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS, |
221 | allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES, | 224 | allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES, |
222 | threads: CONFIG.TRANSCODING.THREADS, | 225 | threads: CONFIG.TRANSCODING.THREADS, |
@@ -252,6 +255,9 @@ function customConfig (): CustomConfig { | |||
252 | maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, | 255 | maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, |
253 | transcoding: { | 256 | transcoding: { |
254 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | 257 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, |
258 | remoteRunners: { | ||
259 | enabled: CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
260 | }, | ||
255 | threads: CONFIG.LIVE.TRANSCODING.THREADS, | 261 | threads: CONFIG.LIVE.TRANSCODING.THREADS, |
256 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, | 262 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, |
257 | resolutions: { | 263 | resolutions: { |
diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts index e1d197c8a..646f9597e 100644 --- a/server/controllers/api/index.ts +++ b/server/controllers/api/index.ts | |||
@@ -15,6 +15,7 @@ import { metricsRouter } from './metrics' | |||
15 | import { oauthClientsRouter } from './oauth-clients' | 15 | import { oauthClientsRouter } from './oauth-clients' |
16 | import { overviewsRouter } from './overviews' | 16 | import { overviewsRouter } from './overviews' |
17 | import { pluginRouter } from './plugins' | 17 | import { pluginRouter } from './plugins' |
18 | import { runnersRouter } from './runners' | ||
18 | import { searchRouter } from './search' | 19 | import { searchRouter } from './search' |
19 | import { serverRouter } from './server' | 20 | import { serverRouter } from './server' |
20 | import { usersRouter } from './users' | 21 | import { usersRouter } from './users' |
@@ -55,6 +56,7 @@ apiRouter.use('/overviews', overviewsRouter) | |||
55 | apiRouter.use('/plugins', pluginRouter) | 56 | apiRouter.use('/plugins', pluginRouter) |
56 | apiRouter.use('/custom-pages', customPageRouter) | 57 | apiRouter.use('/custom-pages', customPageRouter) |
57 | apiRouter.use('/blocklist', blocklistRouter) | 58 | apiRouter.use('/blocklist', blocklistRouter) |
59 | apiRouter.use('/runners', runnersRouter) | ||
58 | apiRouter.use('/ping', pong) | 60 | apiRouter.use('/ping', pong) |
59 | apiRouter.use('/*', badRequest) | 61 | apiRouter.use('/*', badRequest) |
60 | 62 | ||
diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts index 6a53e3083..b63e2f962 100644 --- a/server/controllers/api/jobs.ts +++ b/server/controllers/api/jobs.ts | |||
@@ -93,6 +93,9 @@ async function formatJob (job: BullJob, state?: JobState): Promise<Job> { | |||
93 | state: state || await job.getState(), | 93 | state: state || await job.getState(), |
94 | type: job.queueName as JobType, | 94 | type: job.queueName as JobType, |
95 | data: job.data, | 95 | data: job.data, |
96 | parent: job.parent | ||
97 | ? { id: job.parent.id } | ||
98 | : undefined, | ||
96 | progress: job.progress as number, | 99 | progress: job.progress as number, |
97 | priority: job.opts.priority, | 100 | priority: job.opts.priority, |
98 | error, | 101 | error, |
diff --git a/server/controllers/api/runners/index.ts b/server/controllers/api/runners/index.ts new file mode 100644 index 000000000..c98ded354 --- /dev/null +++ b/server/controllers/api/runners/index.ts | |||
@@ -0,0 +1,18 @@ | |||
1 | import express from 'express' | ||
2 | import { runnerJobsRouter } from './jobs' | ||
3 | import { runnerJobFilesRouter } from './jobs-files' | ||
4 | import { manageRunnersRouter } from './manage-runners' | ||
5 | import { runnerRegistrationTokensRouter } from './registration-tokens' | ||
6 | |||
7 | const runnersRouter = express.Router() | ||
8 | |||
9 | runnersRouter.use('/', manageRunnersRouter) | ||
10 | runnersRouter.use('/', runnerJobsRouter) | ||
11 | runnersRouter.use('/', runnerJobFilesRouter) | ||
12 | runnersRouter.use('/', runnerRegistrationTokensRouter) | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export { | ||
17 | runnersRouter | ||
18 | } | ||
diff --git a/server/controllers/api/runners/jobs-files.ts b/server/controllers/api/runners/jobs-files.ts new file mode 100644 index 000000000..e43ce35f5 --- /dev/null +++ b/server/controllers/api/runners/jobs-files.ts | |||
@@ -0,0 +1,84 @@ | |||
1 | import express from 'express' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' | ||
4 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
5 | import { asyncMiddleware } from '@server/middlewares' | ||
6 | import { jobOfRunnerGetValidator } from '@server/middlewares/validators/runners' | ||
7 | import { runnerJobGetVideoTranscodingFileValidator } from '@server/middlewares/validators/runners/job-files' | ||
8 | import { VideoStorage } from '@shared/models' | ||
9 | |||
10 | const lTags = loggerTagsFactory('api', 'runner') | ||
11 | |||
12 | const runnerJobFilesRouter = express.Router() | ||
13 | |||
14 | runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/max-quality', | ||
15 | asyncMiddleware(jobOfRunnerGetValidator), | ||
16 | asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), | ||
17 | asyncMiddleware(getMaxQualityVideoFile) | ||
18 | ) | ||
19 | |||
20 | runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-quality', | ||
21 | asyncMiddleware(jobOfRunnerGetValidator), | ||
22 | asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), | ||
23 | getMaxQualityVideoPreview | ||
24 | ) | ||
25 | |||
26 | // --------------------------------------------------------------------------- | ||
27 | |||
28 | export { | ||
29 | runnerJobFilesRouter | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | |||
34 | async function getMaxQualityVideoFile (req: express.Request, res: express.Response) { | ||
35 | const runnerJob = res.locals.runnerJob | ||
36 | const runner = runnerJob.Runner | ||
37 | const video = res.locals.videoAll | ||
38 | |||
39 | logger.info( | ||
40 | 'Get max quality file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, | ||
41 | lTags(runner.name, runnerJob.id, runnerJob.type) | ||
42 | ) | ||
43 | |||
44 | const file = video.getMaxQualityFile() | ||
45 | |||
46 | if (file.storage === VideoStorage.OBJECT_STORAGE) { | ||
47 | if (file.isHLS()) { | ||
48 | return proxifyHLS({ | ||
49 | req, | ||
50 | res, | ||
51 | filename: file.filename, | ||
52 | playlist: video.getHLSPlaylist(), | ||
53 | reinjectVideoFileToken: false, | ||
54 | video | ||
55 | }) | ||
56 | } | ||
57 | |||
58 | // Web video | ||
59 | return proxifyWebTorrentFile({ | ||
60 | req, | ||
61 | res, | ||
62 | filename: file.filename | ||
63 | }) | ||
64 | } | ||
65 | |||
66 | return VideoPathManager.Instance.makeAvailableVideoFile(file, videoPath => { | ||
67 | return res.sendFile(videoPath) | ||
68 | }) | ||
69 | } | ||
70 | |||
71 | function getMaxQualityVideoPreview (req: express.Request, res: express.Response) { | ||
72 | const runnerJob = res.locals.runnerJob | ||
73 | const runner = runnerJob.Runner | ||
74 | const video = res.locals.videoAll | ||
75 | |||
76 | logger.info( | ||
77 | 'Get max quality preview file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, | ||
78 | lTags(runner.name, runnerJob.id, runnerJob.type) | ||
79 | ) | ||
80 | |||
81 | const file = video.getPreview() | ||
82 | |||
83 | return res.sendFile(file.getPath()) | ||
84 | } | ||
diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts new file mode 100644 index 000000000..7d488ec11 --- /dev/null +++ b/server/controllers/api/runners/jobs.ts | |||
@@ -0,0 +1,352 @@ | |||
1 | import express, { UploadFiles } from 'express' | ||
2 | import { createReqFiles } from '@server/helpers/express-utils' | ||
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
4 | import { generateRunnerJobToken } from '@server/helpers/token-generator' | ||
5 | import { MIMETYPES } from '@server/initializers/constants' | ||
6 | import { sequelizeTypescript } from '@server/initializers/database' | ||
7 | import { getRunnerJobHandlerClass, updateLastRunnerContact } from '@server/lib/runners' | ||
8 | import { | ||
9 | asyncMiddleware, | ||
10 | authenticate, | ||
11 | ensureUserHasRight, | ||
12 | paginationValidator, | ||
13 | runnerJobsSortValidator, | ||
14 | setDefaultPagination, | ||
15 | setDefaultSort | ||
16 | } from '@server/middlewares' | ||
17 | import { | ||
18 | abortRunnerJobValidator, | ||
19 | acceptRunnerJobValidator, | ||
20 | errorRunnerJobValidator, | ||
21 | getRunnerFromTokenValidator, | ||
22 | jobOfRunnerGetValidator, | ||
23 | runnerJobGetValidator, | ||
24 | successRunnerJobValidator, | ||
25 | updateRunnerJobValidator | ||
26 | } from '@server/middlewares/validators/runners' | ||
27 | import { RunnerModel } from '@server/models/runner/runner' | ||
28 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
29 | import { | ||
30 | AbortRunnerJobBody, | ||
31 | AcceptRunnerJobResult, | ||
32 | ErrorRunnerJobBody, | ||
33 | HttpStatusCode, | ||
34 | ListRunnerJobsQuery, | ||
35 | LiveRTMPHLSTranscodingUpdatePayload, | ||
36 | RequestRunnerJobResult, | ||
37 | RunnerJobState, | ||
38 | RunnerJobSuccessBody, | ||
39 | RunnerJobSuccessPayload, | ||
40 | RunnerJobType, | ||
41 | RunnerJobUpdateBody, | ||
42 | RunnerJobUpdatePayload, | ||
43 | UserRight, | ||
44 | VODAudioMergeTranscodingSuccess, | ||
45 | VODHLSTranscodingSuccess, | ||
46 | VODWebVideoTranscodingSuccess | ||
47 | } from '@shared/models' | ||
48 | |||
49 | const postRunnerJobSuccessVideoFiles = createReqFiles( | ||
50 | [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]' ], | ||
51 | { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } | ||
52 | ) | ||
53 | |||
54 | const runnerJobUpdateVideoFiles = createReqFiles( | ||
55 | [ 'payload[videoChunkFile]', 'payload[resolutionPlaylistFile]', 'payload[masterPlaylistFile]' ], | ||
56 | { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } | ||
57 | ) | ||
58 | |||
59 | const lTags = loggerTagsFactory('api', 'runner') | ||
60 | |||
61 | const runnerJobsRouter = express.Router() | ||
62 | |||
63 | // --------------------------------------------------------------------------- | ||
64 | // Controllers for runners | ||
65 | // --------------------------------------------------------------------------- | ||
66 | |||
67 | runnerJobsRouter.post('/jobs/request', | ||
68 | asyncMiddleware(getRunnerFromTokenValidator), | ||
69 | asyncMiddleware(requestRunnerJob) | ||
70 | ) | ||
71 | |||
72 | runnerJobsRouter.post('/jobs/:jobUUID/accept', | ||
73 | asyncMiddleware(runnerJobGetValidator), | ||
74 | acceptRunnerJobValidator, | ||
75 | asyncMiddleware(getRunnerFromTokenValidator), | ||
76 | asyncMiddleware(acceptRunnerJob) | ||
77 | ) | ||
78 | |||
79 | runnerJobsRouter.post('/jobs/:jobUUID/abort', | ||
80 | asyncMiddleware(jobOfRunnerGetValidator), | ||
81 | abortRunnerJobValidator, | ||
82 | asyncMiddleware(abortRunnerJob) | ||
83 | ) | ||
84 | |||
85 | runnerJobsRouter.post('/jobs/:jobUUID/update', | ||
86 | runnerJobUpdateVideoFiles, | ||
87 | asyncMiddleware(jobOfRunnerGetValidator), | ||
88 | updateRunnerJobValidator, | ||
89 | asyncMiddleware(updateRunnerJobController) | ||
90 | ) | ||
91 | |||
92 | runnerJobsRouter.post('/jobs/:jobUUID/error', | ||
93 | asyncMiddleware(jobOfRunnerGetValidator), | ||
94 | errorRunnerJobValidator, | ||
95 | asyncMiddleware(errorRunnerJob) | ||
96 | ) | ||
97 | |||
98 | runnerJobsRouter.post('/jobs/:jobUUID/success', | ||
99 | postRunnerJobSuccessVideoFiles, | ||
100 | asyncMiddleware(jobOfRunnerGetValidator), | ||
101 | successRunnerJobValidator, | ||
102 | asyncMiddleware(postRunnerJobSuccess) | ||
103 | ) | ||
104 | |||
105 | // --------------------------------------------------------------------------- | ||
106 | // Controllers for admins | ||
107 | // --------------------------------------------------------------------------- | ||
108 | |||
109 | runnerJobsRouter.post('/jobs/:jobUUID/cancel', | ||
110 | authenticate, | ||
111 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
112 | asyncMiddleware(runnerJobGetValidator), | ||
113 | asyncMiddleware(cancelRunnerJob) | ||
114 | ) | ||
115 | |||
116 | runnerJobsRouter.get('/jobs', | ||
117 | authenticate, | ||
118 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
119 | paginationValidator, | ||
120 | runnerJobsSortValidator, | ||
121 | setDefaultSort, | ||
122 | setDefaultPagination, | ||
123 | asyncMiddleware(listRunnerJobs) | ||
124 | ) | ||
125 | |||
126 | // --------------------------------------------------------------------------- | ||
127 | |||
128 | export { | ||
129 | runnerJobsRouter | ||
130 | } | ||
131 | |||
132 | // --------------------------------------------------------------------------- | ||
133 | |||
134 | // --------------------------------------------------------------------------- | ||
135 | // Controllers for runners | ||
136 | // --------------------------------------------------------------------------- | ||
137 | |||
138 | async function requestRunnerJob (req: express.Request, res: express.Response) { | ||
139 | const runner = res.locals.runner | ||
140 | const availableJobs = await RunnerJobModel.listAvailableJobs() | ||
141 | |||
142 | logger.debug('Runner %s requests for a job.', runner.name, { availableJobs, ...lTags(runner.name) }) | ||
143 | |||
144 | const result: RequestRunnerJobResult = { | ||
145 | availableJobs: availableJobs.map(j => ({ | ||
146 | uuid: j.uuid, | ||
147 | type: j.type, | ||
148 | payload: j.payload | ||
149 | })) | ||
150 | } | ||
151 | |||
152 | updateLastRunnerContact(req, runner) | ||
153 | |||
154 | return res.json(result) | ||
155 | } | ||
156 | |||
157 | async function acceptRunnerJob (req: express.Request, res: express.Response) { | ||
158 | const runner = res.locals.runner | ||
159 | const runnerJob = res.locals.runnerJob | ||
160 | |||
161 | runnerJob.state = RunnerJobState.PROCESSING | ||
162 | runnerJob.processingJobToken = generateRunnerJobToken() | ||
163 | runnerJob.startedAt = new Date() | ||
164 | runnerJob.runnerId = runner.id | ||
165 | |||
166 | const newRunnerJob = await sequelizeTypescript.transaction(transaction => { | ||
167 | return runnerJob.save({ transaction }) | ||
168 | }) | ||
169 | newRunnerJob.Runner = runner as RunnerModel | ||
170 | |||
171 | const result: AcceptRunnerJobResult = { | ||
172 | job: { | ||
173 | ...newRunnerJob.toFormattedJSON(), | ||
174 | |||
175 | jobToken: newRunnerJob.processingJobToken | ||
176 | } | ||
177 | } | ||
178 | |||
179 | updateLastRunnerContact(req, runner) | ||
180 | |||
181 | logger.info( | ||
182 | 'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, | ||
183 | lTags(runner.name, runnerJob.uuid, runnerJob.type) | ||
184 | ) | ||
185 | |||
186 | return res.json(result) | ||
187 | } | ||
188 | |||
189 | async function abortRunnerJob (req: express.Request, res: express.Response) { | ||
190 | const runnerJob = res.locals.runnerJob | ||
191 | const runner = runnerJob.Runner | ||
192 | const body: AbortRunnerJobBody = req.body | ||
193 | |||
194 | logger.info( | ||
195 | 'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, | ||
196 | { reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } | ||
197 | ) | ||
198 | |||
199 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) | ||
200 | await new RunnerJobHandler().abort({ runnerJob }) | ||
201 | |||
202 | updateLastRunnerContact(req, runnerJob.Runner) | ||
203 | |||
204 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
205 | } | ||
206 | |||
207 | async function errorRunnerJob (req: express.Request, res: express.Response) { | ||
208 | const runnerJob = res.locals.runnerJob | ||
209 | const runner = runnerJob.Runner | ||
210 | const body: ErrorRunnerJobBody = req.body | ||
211 | |||
212 | runnerJob.failures += 1 | ||
213 | |||
214 | logger.error( | ||
215 | 'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, | ||
216 | { errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } | ||
217 | ) | ||
218 | |||
219 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) | ||
220 | await new RunnerJobHandler().error({ runnerJob, message: body.message }) | ||
221 | |||
222 | updateLastRunnerContact(req, runnerJob.Runner) | ||
223 | |||
224 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
225 | } | ||
226 | |||
227 | // --------------------------------------------------------------------------- | ||
228 | |||
229 | const jobUpdateBuilders: { | ||
230 | [id in RunnerJobType]?: (payload: RunnerJobUpdatePayload, files?: UploadFiles) => RunnerJobUpdatePayload | ||
231 | } = { | ||
232 | 'live-rtmp-hls-transcoding': (payload: LiveRTMPHLSTranscodingUpdatePayload, files) => { | ||
233 | return { | ||
234 | ...payload, | ||
235 | |||
236 | masterPlaylistFile: files['payload[masterPlaylistFile]']?.[0].path, | ||
237 | resolutionPlaylistFile: files['payload[resolutionPlaylistFile]']?.[0].path, | ||
238 | videoChunkFile: files['payload[videoChunkFile]']?.[0].path | ||
239 | } | ||
240 | } | ||
241 | } | ||
242 | |||
243 | async function updateRunnerJobController (req: express.Request, res: express.Response) { | ||
244 | const runnerJob = res.locals.runnerJob | ||
245 | const runner = runnerJob.Runner | ||
246 | const body: RunnerJobUpdateBody = req.body | ||
247 | |||
248 | const payloadBuilder = jobUpdateBuilders[runnerJob.type] | ||
249 | const updatePayload = payloadBuilder | ||
250 | ? payloadBuilder(body.payload, req.files as UploadFiles) | ||
251 | : undefined | ||
252 | |||
253 | logger.debug( | ||
254 | 'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, | ||
255 | { body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } | ||
256 | ) | ||
257 | |||
258 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) | ||
259 | await new RunnerJobHandler().update({ | ||
260 | runnerJob, | ||
261 | progress: req.body.progress, | ||
262 | updatePayload | ||
263 | }) | ||
264 | |||
265 | updateLastRunnerContact(req, runnerJob.Runner) | ||
266 | |||
267 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
268 | } | ||
269 | |||
270 | // --------------------------------------------------------------------------- | ||
271 | |||
272 | const jobSuccessPayloadBuilders: { | ||
273 | [id in RunnerJobType]: (payload: RunnerJobSuccessPayload, files?: UploadFiles) => RunnerJobSuccessPayload | ||
274 | } = { | ||
275 | 'vod-web-video-transcoding': (payload: VODWebVideoTranscodingSuccess, files) => { | ||
276 | return { | ||
277 | ...payload, | ||
278 | |||
279 | videoFile: files['payload[videoFile]'][0].path | ||
280 | } | ||
281 | }, | ||
282 | |||
283 | 'vod-hls-transcoding': (payload: VODHLSTranscodingSuccess, files) => { | ||
284 | return { | ||
285 | ...payload, | ||
286 | |||
287 | videoFile: files['payload[videoFile]'][0].path, | ||
288 | resolutionPlaylistFile: files['payload[resolutionPlaylistFile]'][0].path | ||
289 | } | ||
290 | }, | ||
291 | |||
292 | 'vod-audio-merge-transcoding': (payload: VODAudioMergeTranscodingSuccess, files) => { | ||
293 | return { | ||
294 | ...payload, | ||
295 | |||
296 | videoFile: files['payload[videoFile]'][0].path | ||
297 | } | ||
298 | }, | ||
299 | |||
300 | 'live-rtmp-hls-transcoding': () => ({}) | ||
301 | } | ||
302 | |||
303 | async function postRunnerJobSuccess (req: express.Request, res: express.Response) { | ||
304 | const runnerJob = res.locals.runnerJob | ||
305 | const runner = runnerJob.Runner | ||
306 | const body: RunnerJobSuccessBody = req.body | ||
307 | |||
308 | const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles) | ||
309 | |||
310 | logger.info( | ||
311 | 'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, | ||
312 | { resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } | ||
313 | ) | ||
314 | |||
315 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) | ||
316 | await new RunnerJobHandler().complete({ runnerJob, resultPayload }) | ||
317 | |||
318 | updateLastRunnerContact(req, runnerJob.Runner) | ||
319 | |||
320 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
321 | } | ||
322 | |||
323 | // --------------------------------------------------------------------------- | ||
324 | // Controllers for admins | ||
325 | // --------------------------------------------------------------------------- | ||
326 | |||
327 | async function cancelRunnerJob (req: express.Request, res: express.Response) { | ||
328 | const runnerJob = res.locals.runnerJob | ||
329 | |||
330 | logger.info('Cancelling job %s (%s)', runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) | ||
331 | |||
332 | const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) | ||
333 | await new RunnerJobHandler().cancel({ runnerJob }) | ||
334 | |||
335 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
336 | } | ||
337 | |||
338 | async function listRunnerJobs (req: express.Request, res: express.Response) { | ||
339 | const query: ListRunnerJobsQuery = req.query | ||
340 | |||
341 | const resultList = await RunnerJobModel.listForApi({ | ||
342 | start: query.start, | ||
343 | count: query.count, | ||
344 | sort: query.sort, | ||
345 | search: query.search | ||
346 | }) | ||
347 | |||
348 | return res.json({ | ||
349 | total: resultList.total, | ||
350 | data: resultList.data.map(d => d.toFormattedAdminJSON()) | ||
351 | }) | ||
352 | } | ||
diff --git a/server/controllers/api/runners/manage-runners.ts b/server/controllers/api/runners/manage-runners.ts new file mode 100644 index 000000000..eb08c4b1d --- /dev/null +++ b/server/controllers/api/runners/manage-runners.ts | |||
@@ -0,0 +1,107 @@ | |||
1 | import express from 'express' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { generateRunnerToken } from '@server/helpers/token-generator' | ||
4 | import { | ||
5 | asyncMiddleware, | ||
6 | authenticate, | ||
7 | ensureUserHasRight, | ||
8 | paginationValidator, | ||
9 | runnersSortValidator, | ||
10 | setDefaultPagination, | ||
11 | setDefaultSort | ||
12 | } from '@server/middlewares' | ||
13 | import { deleteRunnerValidator, getRunnerFromTokenValidator, registerRunnerValidator } from '@server/middlewares/validators/runners' | ||
14 | import { RunnerModel } from '@server/models/runner/runner' | ||
15 | import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@shared/models' | ||
16 | |||
17 | const lTags = loggerTagsFactory('api', 'runner') | ||
18 | |||
19 | const manageRunnersRouter = express.Router() | ||
20 | |||
21 | manageRunnersRouter.post('/register', | ||
22 | asyncMiddleware(registerRunnerValidator), | ||
23 | asyncMiddleware(registerRunner) | ||
24 | ) | ||
25 | manageRunnersRouter.post('/unregister', | ||
26 | asyncMiddleware(getRunnerFromTokenValidator), | ||
27 | asyncMiddleware(unregisterRunner) | ||
28 | ) | ||
29 | |||
30 | manageRunnersRouter.delete('/:runnerId', | ||
31 | authenticate, | ||
32 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
33 | asyncMiddleware(deleteRunnerValidator), | ||
34 | asyncMiddleware(deleteRunner) | ||
35 | ) | ||
36 | |||
37 | manageRunnersRouter.get('/', | ||
38 | authenticate, | ||
39 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
40 | paginationValidator, | ||
41 | runnersSortValidator, | ||
42 | setDefaultSort, | ||
43 | setDefaultPagination, | ||
44 | asyncMiddleware(listRunners) | ||
45 | ) | ||
46 | |||
47 | // --------------------------------------------------------------------------- | ||
48 | |||
49 | export { | ||
50 | manageRunnersRouter | ||
51 | } | ||
52 | |||
53 | // --------------------------------------------------------------------------- | ||
54 | |||
55 | async function registerRunner (req: express.Request, res: express.Response) { | ||
56 | const body: RegisterRunnerBody = req.body | ||
57 | |||
58 | const runnerToken = generateRunnerToken() | ||
59 | |||
60 | const runner = new RunnerModel({ | ||
61 | runnerToken, | ||
62 | name: body.name, | ||
63 | description: body.description, | ||
64 | lastContact: new Date(), | ||
65 | ip: req.ip, | ||
66 | runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id | ||
67 | }) | ||
68 | |||
69 | await runner.save() | ||
70 | |||
71 | logger.info('Registered new runner %s', runner.name, { ...lTags(runner.name) }) | ||
72 | |||
73 | return res.json({ id: runner.id, runnerToken }) | ||
74 | } | ||
75 | async function unregisterRunner (req: express.Request, res: express.Response) { | ||
76 | const runner = res.locals.runner | ||
77 | await runner.destroy() | ||
78 | |||
79 | logger.info('Unregistered runner %s', runner.name, { ...lTags(runner.name) }) | ||
80 | |||
81 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
82 | } | ||
83 | |||
84 | async function deleteRunner (req: express.Request, res: express.Response) { | ||
85 | const runner = res.locals.runner | ||
86 | |||
87 | await runner.destroy() | ||
88 | |||
89 | logger.info('Deleted runner %s', runner.name, { ...lTags(runner.name) }) | ||
90 | |||
91 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
92 | } | ||
93 | |||
94 | async function listRunners (req: express.Request, res: express.Response) { | ||
95 | const query: ListRunnersQuery = req.query | ||
96 | |||
97 | const resultList = await RunnerModel.listForApi({ | ||
98 | start: query.start, | ||
99 | count: query.count, | ||
100 | sort: query.sort | ||
101 | }) | ||
102 | |||
103 | return res.json({ | ||
104 | total: resultList.total, | ||
105 | data: resultList.data.map(d => d.toFormattedJSON()) | ||
106 | }) | ||
107 | } | ||
diff --git a/server/controllers/api/runners/registration-tokens.ts b/server/controllers/api/runners/registration-tokens.ts new file mode 100644 index 000000000..5ac3773fe --- /dev/null +++ b/server/controllers/api/runners/registration-tokens.ts | |||
@@ -0,0 +1,87 @@ | |||
1 | import express from 'express' | ||
2 | import { generateRunnerRegistrationToken } from '@server/helpers/token-generator' | ||
3 | import { | ||
4 | asyncMiddleware, | ||
5 | authenticate, | ||
6 | ensureUserHasRight, | ||
7 | paginationValidator, | ||
8 | runnerRegistrationTokensSortValidator, | ||
9 | setDefaultPagination, | ||
10 | setDefaultSort | ||
11 | } from '@server/middlewares' | ||
12 | import { deleteRegistrationTokenValidator } from '@server/middlewares/validators/runners' | ||
13 | import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' | ||
14 | import { HttpStatusCode, ListRunnerRegistrationTokensQuery, UserRight } from '@shared/models' | ||
15 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
16 | |||
17 | const lTags = loggerTagsFactory('api', 'runner') | ||
18 | |||
19 | const runnerRegistrationTokensRouter = express.Router() | ||
20 | |||
21 | runnerRegistrationTokensRouter.post('/registration-tokens/generate', | ||
22 | authenticate, | ||
23 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
24 | asyncMiddleware(generateRegistrationToken) | ||
25 | ) | ||
26 | |||
27 | runnerRegistrationTokensRouter.delete('/registration-tokens/:id', | ||
28 | authenticate, | ||
29 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
30 | asyncMiddleware(deleteRegistrationTokenValidator), | ||
31 | asyncMiddleware(deleteRegistrationToken) | ||
32 | ) | ||
33 | |||
34 | runnerRegistrationTokensRouter.get('/registration-tokens', | ||
35 | authenticate, | ||
36 | ensureUserHasRight(UserRight.MANAGE_RUNNERS), | ||
37 | paginationValidator, | ||
38 | runnerRegistrationTokensSortValidator, | ||
39 | setDefaultSort, | ||
40 | setDefaultPagination, | ||
41 | asyncMiddleware(listRegistrationTokens) | ||
42 | ) | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | export { | ||
47 | runnerRegistrationTokensRouter | ||
48 | } | ||
49 | |||
50 | // --------------------------------------------------------------------------- | ||
51 | |||
52 | async function generateRegistrationToken (req: express.Request, res: express.Response) { | ||
53 | logger.info('Generating new runner registration token.', lTags()) | ||
54 | |||
55 | const registrationToken = new RunnerRegistrationTokenModel({ | ||
56 | registrationToken: generateRunnerRegistrationToken() | ||
57 | }) | ||
58 | |||
59 | await registrationToken.save() | ||
60 | |||
61 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
62 | } | ||
63 | |||
64 | async function deleteRegistrationToken (req: express.Request, res: express.Response) { | ||
65 | logger.info('Removing runner registration token.', lTags()) | ||
66 | |||
67 | const runnerRegistrationToken = res.locals.runnerRegistrationToken | ||
68 | |||
69 | await runnerRegistrationToken.destroy() | ||
70 | |||
71 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
72 | } | ||
73 | |||
74 | async function listRegistrationTokens (req: express.Request, res: express.Response) { | ||
75 | const query: ListRunnerRegistrationTokensQuery = req.query | ||
76 | |||
77 | const resultList = await RunnerRegistrationTokenModel.listForApi({ | ||
78 | start: query.start, | ||
79 | count: query.count, | ||
80 | sort: query.sort | ||
81 | }) | ||
82 | |||
83 | return res.json({ | ||
84 | total: resultList.total, | ||
85 | data: resultList.data.map(d => d.toFormattedJSON()) | ||
86 | }) | ||
87 | } | ||
diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts index 8c9a5322b..54f484b2b 100644 --- a/server/controllers/api/videos/transcoding.ts +++ b/server/controllers/api/videos/transcoding.ts | |||
@@ -1,10 +1,8 @@ | |||
1 | import Bluebird from 'bluebird' | ||
2 | import express from 'express' | 1 | import express from 'express' |
3 | import { computeResolutionsToTranscode } from '@server/helpers/ffmpeg' | ||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { JobQueue } from '@server/lib/job-queue' | ||
6 | import { Hooks } from '@server/lib/plugins/hooks' | 3 | import { Hooks } from '@server/lib/plugins/hooks' |
7 | import { buildTranscodingJob } from '@server/lib/video' | 4 | import { createTranscodingJobs } from '@server/lib/transcoding/create-transcoding-job' |
5 | import { computeResolutionsToTranscode } from '@server/lib/transcoding/transcoding-resolutions' | ||
8 | import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models' | 6 | import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models' |
9 | import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares' | 7 | import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares' |
10 | 8 | ||
@@ -47,82 +45,13 @@ async function createTranscoding (req: express.Request, res: express.Response) { | |||
47 | video.state = VideoState.TO_TRANSCODE | 45 | video.state = VideoState.TO_TRANSCODE |
48 | await video.save() | 46 | await video.save() |
49 | 47 | ||
50 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | 48 | await createTranscodingJobs({ |
51 | 49 | video, | |
52 | logger.info('Manually creating transcoding jobs for %s.', body.transcodingType, { childrenResolutions, maxResolution }) | 50 | resolutions, |
53 | 51 | transcodingType: body.transcodingType, | |
54 | const children = await Bluebird.mapSeries(childrenResolutions, resolution => { | ||
55 | if (body.transcodingType === 'hls') { | ||
56 | return buildHLSJobOption({ | ||
57 | videoUUID: video.uuid, | ||
58 | hasAudio, | ||
59 | resolution, | ||
60 | isMaxQuality: false | ||
61 | }) | ||
62 | } | ||
63 | |||
64 | if (body.transcodingType === 'webtorrent') { | ||
65 | return buildWebTorrentJobOption({ | ||
66 | videoUUID: video.uuid, | ||
67 | hasAudio, | ||
68 | resolution | ||
69 | }) | ||
70 | } | ||
71 | }) | ||
72 | |||
73 | const parent = body.transcodingType === 'hls' | ||
74 | ? await buildHLSJobOption({ | ||
75 | videoUUID: video.uuid, | ||
76 | hasAudio, | ||
77 | resolution: maxResolution, | ||
78 | isMaxQuality: false | ||
79 | }) | ||
80 | : await buildWebTorrentJobOption({ | ||
81 | videoUUID: video.uuid, | ||
82 | hasAudio, | ||
83 | resolution: maxResolution | ||
84 | }) | ||
85 | |||
86 | // Porcess the last resolution after the other ones to prevent concurrency issue | ||
87 | // Because low resolutions use the biggest one as ffmpeg input | ||
88 | await JobQueue.Instance.createJobWithChildren(parent, children) | ||
89 | |||
90 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) | ||
91 | } | ||
92 | |||
93 | function buildHLSJobOption (options: { | ||
94 | videoUUID: string | ||
95 | hasAudio: boolean | ||
96 | resolution: number | ||
97 | isMaxQuality: boolean | ||
98 | }) { | ||
99 | const { videoUUID, hasAudio, resolution, isMaxQuality } = options | ||
100 | |||
101 | return buildTranscodingJob({ | ||
102 | type: 'new-resolution-to-hls', | ||
103 | videoUUID, | ||
104 | resolution, | ||
105 | hasAudio, | ||
106 | copyCodecs: false, | ||
107 | isNewVideo: false, | 52 | isNewVideo: false, |
108 | autoDeleteWebTorrentIfNeeded: false, | 53 | user: null // Don't specify priority since these transcoding jobs are fired by the admin |
109 | isMaxQuality | ||
110 | }) | 54 | }) |
111 | } | ||
112 | 55 | ||
113 | function buildWebTorrentJobOption (options: { | 56 | return res.sendStatus(HttpStatusCode.NO_CONTENT_204) |
114 | videoUUID: string | ||
115 | hasAudio: boolean | ||
116 | resolution: number | ||
117 | }) { | ||
118 | const { videoUUID, hasAudio, resolution } = options | ||
119 | |||
120 | return buildTranscodingJob({ | ||
121 | type: 'new-resolution-to-webtorrent', | ||
122 | videoUUID, | ||
123 | isNewVideo: false, | ||
124 | resolution, | ||
125 | hasAudio, | ||
126 | createHLSIfNeeded: false | ||
127 | }) | ||
128 | } | 57 | } |
diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 43313a143..885ac8b81 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts | |||
@@ -3,28 +3,20 @@ import { move } from 'fs-extra' | |||
3 | import { basename } from 'path' | 3 | import { basename } from 'path' |
4 | import { getResumableUploadPath } from '@server/helpers/upload' | 4 | import { getResumableUploadPath } from '@server/helpers/upload' |
5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
6 | import { JobQueue } from '@server/lib/job-queue' | 6 | import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue' |
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | ||
8 | import { Redis } from '@server/lib/redis' | 7 | import { Redis } from '@server/lib/redis' |
9 | import { uploadx } from '@server/lib/uploadx' | 8 | import { uploadx } from '@server/lib/uploadx' |
10 | import { | 9 | import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' |
11 | buildLocalVideoFromReq, | 10 | import { buildNewFile } from '@server/lib/video-file' |
12 | buildMoveToObjectStorageJob, | ||
13 | buildOptimizeOrMergeAudioJob, | ||
14 | buildVideoThumbnailsFromReq, | ||
15 | setVideoTags | ||
16 | } from '@server/lib/video' | ||
17 | import { VideoPathManager } from '@server/lib/video-path-manager' | 11 | import { VideoPathManager } from '@server/lib/video-path-manager' |
18 | import { buildNextVideoState } from '@server/lib/video-state' | 12 | import { buildNextVideoState } from '@server/lib/video-state' |
19 | import { openapiOperationDoc } from '@server/middlewares/doc' | 13 | import { openapiOperationDoc } from '@server/middlewares/doc' |
20 | import { VideoSourceModel } from '@server/models/video/video-source' | 14 | import { VideoSourceModel } from '@server/models/video/video-source' |
21 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | 15 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' |
22 | import { getLowercaseExtension } from '@shared/core-utils' | 16 | import { uuidToShort } from '@shared/extra-utils' |
23 | import { isAudioFile, uuidToShort } from '@shared/extra-utils' | 17 | import { HttpStatusCode, VideoCreate, VideoState } from '@shared/models' |
24 | import { HttpStatusCode, VideoCreate, VideoResolution, VideoState } from '@shared/models' | ||
25 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' | 18 | import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' |
26 | import { createReqFiles } from '../../../helpers/express-utils' | 19 | import { createReqFiles } from '../../../helpers/express-utils' |
27 | import { buildFileMetadata, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '../../../helpers/ffmpeg' | ||
28 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 20 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
29 | import { MIMETYPES } from '../../../initializers/constants' | 21 | import { MIMETYPES } from '../../../initializers/constants' |
30 | import { sequelizeTypescript } from '../../../initializers/database' | 22 | import { sequelizeTypescript } from '../../../initializers/database' |
@@ -41,7 +33,6 @@ import { | |||
41 | } from '../../../middlewares' | 33 | } from '../../../middlewares' |
42 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' | 34 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' |
43 | import { VideoModel } from '../../../models/video/video' | 35 | import { VideoModel } from '../../../models/video/video' |
44 | import { VideoFileModel } from '../../../models/video/video-file' | ||
45 | 36 | ||
46 | const lTags = loggerTagsFactory('api', 'video') | 37 | const lTags = loggerTagsFactory('api', 'video') |
47 | const auditLogger = auditLoggerFactory('videos') | 38 | const auditLogger = auditLoggerFactory('videos') |
@@ -148,7 +139,7 @@ async function addVideo (options: { | |||
148 | video.VideoChannel = videoChannel | 139 | video.VideoChannel = videoChannel |
149 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object | 140 | video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object |
150 | 141 | ||
151 | const videoFile = await buildNewFile(videoPhysicalFile) | 142 | const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) |
152 | const originalFilename = videoPhysicalFile.originalname | 143 | const originalFilename = videoPhysicalFile.originalname |
153 | 144 | ||
154 | // Move physical file | 145 | // Move physical file |
@@ -227,30 +218,8 @@ async function addVideo (options: { | |||
227 | } | 218 | } |
228 | } | 219 | } |
229 | 220 | ||
230 | async function buildNewFile (videoPhysicalFile: express.VideoUploadFile) { | ||
231 | const videoFile = new VideoFileModel({ | ||
232 | extname: getLowercaseExtension(videoPhysicalFile.filename), | ||
233 | size: videoPhysicalFile.size, | ||
234 | videoStreamingPlaylistId: null, | ||
235 | metadata: await buildFileMetadata(videoPhysicalFile.path) | ||
236 | }) | ||
237 | |||
238 | const probe = await ffprobePromise(videoPhysicalFile.path) | ||
239 | |||
240 | if (await isAudioFile(videoPhysicalFile.path, probe)) { | ||
241 | videoFile.resolution = VideoResolution.H_NOVIDEO | ||
242 | } else { | ||
243 | videoFile.fps = await getVideoStreamFPS(videoPhysicalFile.path, probe) | ||
244 | videoFile.resolution = (await getVideoStreamDimensionsInfo(videoPhysicalFile.path, probe)).resolution | ||
245 | } | ||
246 | |||
247 | videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) | ||
248 | |||
249 | return videoFile | ||
250 | } | ||
251 | |||
252 | async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) { | 221 | async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) { |
253 | return JobQueue.Instance.createSequentialJobFlow( | 222 | const jobs: (CreateJobArgument & CreateJobOptions)[] = [ |
254 | { | 223 | { |
255 | type: 'manage-video-torrent' as 'manage-video-torrent', | 224 | type: 'manage-video-torrent' as 'manage-video-torrent', |
256 | payload: { | 225 | payload: { |
@@ -274,16 +243,26 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide | |||
274 | videoUUID: video.uuid, | 243 | videoUUID: video.uuid, |
275 | isNewVideo: true | 244 | isNewVideo: true |
276 | } | 245 | } |
277 | }, | 246 | } |
247 | ] | ||
278 | 248 | ||
279 | video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE | 249 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { |
280 | ? await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }) | 250 | jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined })) |
281 | : undefined, | 251 | } |
252 | |||
253 | if (video.state === VideoState.TO_TRANSCODE) { | ||
254 | jobs.push({ | ||
255 | type: 'transcoding-job-builder' as 'transcoding-job-builder', | ||
256 | payload: { | ||
257 | videoUUID: video.uuid, | ||
258 | optimizeJob: { | ||
259 | isNewVideo: true | ||
260 | } | ||
261 | } | ||
262 | }) | ||
263 | } | ||
282 | 264 | ||
283 | video.state === VideoState.TO_TRANSCODE | 265 | return JobQueue.Instance.createSequentialJobFlow(...jobs) |
284 | ? await buildOptimizeOrMergeAudioJob({ video, videoFile, user }) | ||
285 | : undefined | ||
286 | ) | ||
287 | } | 266 | } |
288 | 267 | ||
289 | async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) { | 268 | async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) { |
diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts index a5ce1d79f..2b825a730 100644 --- a/server/controllers/bots.ts +++ b/server/controllers/bots.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import { getServerActor } from '@server/models/application/application' | ||
2 | import { logger } from '@uploadx/core' | ||
3 | import express from 'express' | 1 | import express from 'express' |
4 | import { truncate } from 'lodash' | 2 | import { truncate } from 'lodash' |
5 | import { SitemapStream, streamToPromise, ErrorLevel } from 'sitemap' | 3 | import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap' |
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { getServerActor } from '@server/models/application/application' | ||
6 | import { buildNSFWFilter } from '../helpers/express-utils' | 6 | import { buildNSFWFilter } from '../helpers/express-utils' |
7 | import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' | 7 | import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' |
8 | import { asyncMiddleware } from '../middlewares' | 8 | import { asyncMiddleware } from '../middlewares' |
diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts index c530b57f8..8e2cc4af9 100644 --- a/server/controllers/object-storage-proxy.ts +++ b/server/controllers/object-storage-proxy.ts | |||
@@ -1,11 +1,7 @@ | |||
1 | import cors from 'cors' | 1 | import cors from 'cors' |
2 | import express from 'express' | 2 | import express from 'express' |
3 | import { PassThrough, pipeline } from 'stream' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { StreamReplacer } from '@server/helpers/stream-replacer' | ||
6 | import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' | 3 | import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' |
7 | import { injectQueryToPlaylistUrls } from '@server/lib/hls' | 4 | import { proxifyHLS, proxifyWebTorrentFile } from '@server/lib/object-storage' |
8 | import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' | ||
9 | import { | 5 | import { |
10 | asyncMiddleware, | 6 | asyncMiddleware, |
11 | ensureCanAccessPrivateVideoHLSFiles, | 7 | ensureCanAccessPrivateVideoHLSFiles, |
@@ -13,9 +9,7 @@ import { | |||
13 | ensurePrivateObjectStorageProxyIsEnabled, | 9 | ensurePrivateObjectStorageProxyIsEnabled, |
14 | optionalAuthenticate | 10 | optionalAuthenticate |
15 | } from '@server/middlewares' | 11 | } from '@server/middlewares' |
16 | import { HttpStatusCode } from '@shared/models' | 12 | import { doReinjectVideoFileToken } from './shared/m3u8-playlist' |
17 | import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' | ||
18 | import { GetObjectCommandOutput } from '@aws-sdk/client-s3' | ||
19 | 13 | ||
20 | const objectStorageProxyRouter = express.Router() | 14 | const objectStorageProxyRouter = express.Router() |
21 | 15 | ||
@@ -25,14 +19,14 @@ objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':file | |||
25 | ensurePrivateObjectStorageProxyIsEnabled, | 19 | ensurePrivateObjectStorageProxyIsEnabled, |
26 | optionalAuthenticate, | 20 | optionalAuthenticate, |
27 | asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), | 21 | asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), |
28 | asyncMiddleware(proxifyWebTorrent) | 22 | asyncMiddleware(proxifyWebTorrentController) |
29 | ) | 23 | ) |
30 | 24 | ||
31 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', | 25 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', |
32 | ensurePrivateObjectStorageProxyIsEnabled, | 26 | ensurePrivateObjectStorageProxyIsEnabled, |
33 | optionalAuthenticate, | 27 | optionalAuthenticate, |
34 | asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), | 28 | asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), |
35 | asyncMiddleware(proxifyHLS) | 29 | asyncMiddleware(proxifyHLSController) |
36 | ) | 30 | ) |
37 | 31 | ||
38 | // --------------------------------------------------------------------------- | 32 | // --------------------------------------------------------------------------- |
@@ -41,76 +35,25 @@ export { | |||
41 | objectStorageProxyRouter | 35 | objectStorageProxyRouter |
42 | } | 36 | } |
43 | 37 | ||
44 | async function proxifyWebTorrent (req: express.Request, res: express.Response) { | 38 | function proxifyWebTorrentController (req: express.Request, res: express.Response) { |
45 | const filename = req.params.filename | 39 | const filename = req.params.filename |
46 | 40 | ||
47 | logger.debug('Proxifying WebTorrent file %s from object storage.', filename) | 41 | return proxifyWebTorrentFile({ req, res, filename }) |
48 | |||
49 | try { | ||
50 | const { response: s3Response, stream } = await getWebTorrentFileReadStream({ | ||
51 | filename, | ||
52 | rangeHeader: req.header('range') | ||
53 | }) | ||
54 | |||
55 | setS3Headers(res, s3Response) | ||
56 | |||
57 | return stream.pipe(res) | ||
58 | } catch (err) { | ||
59 | return handleObjectStorageFailure(res, err) | ||
60 | } | ||
61 | } | 42 | } |
62 | 43 | ||
63 | async function proxifyHLS (req: express.Request, res: express.Response) { | 44 | function proxifyHLSController (req: express.Request, res: express.Response) { |
64 | const playlist = res.locals.videoStreamingPlaylist | 45 | const playlist = res.locals.videoStreamingPlaylist |
65 | const video = res.locals.onlyVideo | 46 | const video = res.locals.onlyVideo |
66 | const filename = req.params.filename | 47 | const filename = req.params.filename |
67 | 48 | ||
68 | logger.debug('Proxifying HLS file %s from object storage.', filename) | 49 | const reinjectVideoFileToken = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) |
69 | |||
70 | try { | ||
71 | const { response: s3Response, stream } = await getHLSFileReadStream({ | ||
72 | playlist: playlist.withVideo(video), | ||
73 | filename, | ||
74 | rangeHeader: req.header('range') | ||
75 | }) | ||
76 | |||
77 | setS3Headers(res, s3Response) | ||
78 | |||
79 | const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) | ||
80 | ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) | ||
81 | : new PassThrough() | ||
82 | 50 | ||
83 | return pipeline( | 51 | return proxifyHLS({ |
84 | stream, | 52 | req, |
85 | streamReplacer, | 53 | res, |
86 | res, | 54 | playlist, |
87 | err => { | 55 | video, |
88 | if (!err) return | 56 | filename, |
89 | 57 | reinjectVideoFileToken | |
90 | handleObjectStorageFailure(res, err) | ||
91 | } | ||
92 | ) | ||
93 | } catch (err) { | ||
94 | return handleObjectStorageFailure(res, err) | ||
95 | } | ||
96 | } | ||
97 | |||
98 | function handleObjectStorageFailure (res: express.Response, err: Error) { | ||
99 | if (err.name === 'NoSuchKey') { | ||
100 | logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) | ||
101 | return res.sendStatus(HttpStatusCode.NOT_FOUND_404) | ||
102 | } | ||
103 | |||
104 | return res.fail({ | ||
105 | status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, | ||
106 | message: err.message, | ||
107 | type: err.name | ||
108 | }) | 58 | }) |
109 | } | 59 | } |
110 | |||
111 | function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { | ||
112 | if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { | ||
113 | res.setHeader('Content-Range', s3Response.ContentRange) | ||
114 | res.status(HttpStatusCode.PARTIAL_CONTENT_206) | ||
115 | } | ||
116 | } | ||
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 73bd994c1..242c49e89 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -11,6 +11,7 @@ import { truncate } from 'lodash' | |||
11 | import { pipeline } from 'stream' | 11 | import { pipeline } from 'stream' |
12 | import { URL } from 'url' | 12 | import { URL } from 'url' |
13 | import { promisify } from 'util' | 13 | import { promisify } from 'util' |
14 | import { promisify1, promisify2, promisify3 } from '@shared/core-utils' | ||
14 | 15 | ||
15 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { | 16 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { |
16 | if (!oldObject || typeof oldObject !== 'object') { | 17 | if (!oldObject || typeof oldObject !== 'object') { |
@@ -229,18 +230,6 @@ function execShell (command: string, options?: ExecOptions) { | |||
229 | 230 | ||
230 | // --------------------------------------------------------------------------- | 231 | // --------------------------------------------------------------------------- |
231 | 232 | ||
232 | function isOdd (num: number) { | ||
233 | return (num % 2) !== 0 | ||
234 | } | ||
235 | |||
236 | function toEven (num: number) { | ||
237 | if (isOdd(num)) return num + 1 | ||
238 | |||
239 | return num | ||
240 | } | ||
241 | |||
242 | // --------------------------------------------------------------------------- | ||
243 | |||
244 | function generateRSAKeyPairPromise (size: number) { | 233 | function generateRSAKeyPairPromise (size: number) { |
245 | return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { | 234 | return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { |
246 | const options: RSAKeyPairOptions<'pem', 'pem'> = { | 235 | const options: RSAKeyPairOptions<'pem', 'pem'> = { |
@@ -286,40 +275,6 @@ function generateED25519KeyPairPromise () { | |||
286 | 275 | ||
287 | // --------------------------------------------------------------------------- | 276 | // --------------------------------------------------------------------------- |
288 | 277 | ||
289 | function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { | ||
290 | return function promisified (): Promise<A> { | ||
291 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
292 | func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
293 | }) | ||
294 | } | ||
295 | } | ||
296 | |||
297 | // Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2 | ||
298 | function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise<A> { | ||
299 | return function promisified (arg: T): Promise<A> { | ||
300 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
301 | func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
302 | }) | ||
303 | } | ||
304 | } | ||
305 | |||
306 | function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> { | ||
307 | return function promisified (arg1: T, arg2: U): Promise<A> { | ||
308 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
309 | func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
310 | }) | ||
311 | } | ||
312 | } | ||
313 | |||
314 | // eslint-disable-next-line max-len | ||
315 | function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> { | ||
316 | return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> { | ||
317 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
318 | func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
319 | }) | ||
320 | } | ||
321 | } | ||
322 | |||
323 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) | 278 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) |
324 | const scryptPromise = promisify3<string, string, number, Buffer>(scrypt) | 279 | const scryptPromise = promisify3<string, string, number, Buffer>(scrypt) |
325 | const execPromise2 = promisify2<string, any, string>(exec) | 280 | const execPromise2 = promisify2<string, any, string>(exec) |
@@ -345,10 +300,6 @@ export { | |||
345 | pageToStartAndCount, | 300 | pageToStartAndCount, |
346 | peertubeTruncate, | 301 | peertubeTruncate, |
347 | 302 | ||
348 | promisify0, | ||
349 | promisify1, | ||
350 | promisify2, | ||
351 | |||
352 | scryptPromise, | 303 | scryptPromise, |
353 | 304 | ||
354 | randomBytesPromise, | 305 | randomBytesPromise, |
@@ -360,8 +311,5 @@ export { | |||
360 | execPromise, | 311 | execPromise, |
361 | pipelinePromise, | 312 | pipelinePromise, |
362 | 313 | ||
363 | parseSemVersion, | 314 | parseSemVersion |
364 | |||
365 | isOdd, | ||
366 | toEven | ||
367 | } | 315 | } |
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index ebab4c6b2..fa0f469f6 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts | |||
@@ -15,6 +15,10 @@ function isSafePath (p: string) { | |||
15 | }) | 15 | }) |
16 | } | 16 | } |
17 | 17 | ||
18 | function isSafeFilename (filename: string, extension: string) { | ||
19 | return typeof filename === 'string' && !!filename.match(new RegExp(`^[a-z0-9-]+\\.${extension}$`)) | ||
20 | } | ||
21 | |||
18 | function isSafePeerTubeFilenameWithoutExtension (filename: string) { | 22 | function isSafePeerTubeFilenameWithoutExtension (filename: string) { |
19 | return filename.match(/^[a-z0-9-]+$/) | 23 | return filename.match(/^[a-z0-9-]+$/) |
20 | } | 24 | } |
@@ -177,5 +181,6 @@ export { | |||
177 | toIntArray, | 181 | toIntArray, |
178 | isFileValid, | 182 | isFileValid, |
179 | isSafePeerTubeFilenameWithoutExtension, | 183 | isSafePeerTubeFilenameWithoutExtension, |
184 | isSafeFilename, | ||
180 | checkMimetypeRegex | 185 | checkMimetypeRegex |
181 | } | 186 | } |
diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts new file mode 100644 index 000000000..5f755d5bb --- /dev/null +++ b/server/helpers/custom-validators/runners/jobs.ts | |||
@@ -0,0 +1,166 @@ | |||
1 | import { UploadFilesForCheck } from 'express' | ||
2 | import validator from 'validator' | ||
3 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
4 | import { | ||
5 | LiveRTMPHLSTranscodingSuccess, | ||
6 | RunnerJobSuccessPayload, | ||
7 | RunnerJobType, | ||
8 | RunnerJobUpdatePayload, | ||
9 | VODAudioMergeTranscodingSuccess, | ||
10 | VODHLSTranscodingSuccess, | ||
11 | VODWebVideoTranscodingSuccess | ||
12 | } from '@shared/models' | ||
13 | import { exists, isFileValid, isSafeFilename } from '../misc' | ||
14 | |||
15 | const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS | ||
16 | |||
17 | const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ]) | ||
18 | function isRunnerJobTypeValid (value: RunnerJobType) { | ||
19 | return runnerJobTypes.has(value) | ||
20 | } | ||
21 | |||
22 | function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) { | ||
23 | return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || | ||
24 | isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || | ||
25 | isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || | ||
26 | isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) | ||
27 | } | ||
28 | |||
29 | // --------------------------------------------------------------------------- | ||
30 | |||
31 | function isRunnerJobProgressValid (value: string) { | ||
32 | return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS) | ||
33 | } | ||
34 | |||
35 | function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { | ||
36 | return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || | ||
37 | isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || | ||
38 | isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || | ||
39 | isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) | ||
40 | } | ||
41 | |||
42 | // --------------------------------------------------------------------------- | ||
43 | |||
44 | function isRunnerJobTokenValid (value: string) { | ||
45 | return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN) | ||
46 | } | ||
47 | |||
48 | function isRunnerJobAbortReasonValid (value: string) { | ||
49 | return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON) | ||
50 | } | ||
51 | |||
52 | function isRunnerJobErrorMessageValid (value: string) { | ||
53 | return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE) | ||
54 | } | ||
55 | |||
56 | // --------------------------------------------------------------------------- | ||
57 | |||
58 | export { | ||
59 | isRunnerJobTypeValid, | ||
60 | isRunnerJobSuccessPayloadValid, | ||
61 | isRunnerJobUpdatePayloadValid, | ||
62 | isRunnerJobTokenValid, | ||
63 | isRunnerJobErrorMessageValid, | ||
64 | isRunnerJobProgressValid, | ||
65 | isRunnerJobAbortReasonValid | ||
66 | } | ||
67 | |||
68 | // --------------------------------------------------------------------------- | ||
69 | |||
70 | function isRunnerJobVODWebVideoResultPayloadValid ( | ||
71 | _value: VODWebVideoTranscodingSuccess, | ||
72 | type: RunnerJobType, | ||
73 | files: UploadFilesForCheck | ||
74 | ) { | ||
75 | return type === 'vod-web-video-transcoding' && | ||
76 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) | ||
77 | } | ||
78 | |||
79 | function isRunnerJobVODHLSResultPayloadValid ( | ||
80 | _value: VODHLSTranscodingSuccess, | ||
81 | type: RunnerJobType, | ||
82 | files: UploadFilesForCheck | ||
83 | ) { | ||
84 | return type === 'vod-hls-transcoding' && | ||
85 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) && | ||
86 | isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null }) | ||
87 | } | ||
88 | |||
89 | function isRunnerJobVODAudioMergeResultPayloadValid ( | ||
90 | _value: VODAudioMergeTranscodingSuccess, | ||
91 | type: RunnerJobType, | ||
92 | files: UploadFilesForCheck | ||
93 | ) { | ||
94 | return type === 'vod-audio-merge-transcoding' && | ||
95 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) | ||
96 | } | ||
97 | |||
98 | function isRunnerJobLiveRTMPHLSResultPayloadValid ( | ||
99 | value: LiveRTMPHLSTranscodingSuccess, | ||
100 | type: RunnerJobType | ||
101 | ) { | ||
102 | return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
103 | } | ||
104 | |||
105 | // --------------------------------------------------------------------------- | ||
106 | |||
107 | function isRunnerJobVODWebVideoUpdatePayloadValid ( | ||
108 | value: RunnerJobUpdatePayload, | ||
109 | type: RunnerJobType, | ||
110 | _files: UploadFilesForCheck | ||
111 | ) { | ||
112 | return type === 'vod-web-video-transcoding' && | ||
113 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
114 | } | ||
115 | |||
116 | function isRunnerJobVODHLSUpdatePayloadValid ( | ||
117 | value: RunnerJobUpdatePayload, | ||
118 | type: RunnerJobType, | ||
119 | _files: UploadFilesForCheck | ||
120 | ) { | ||
121 | return type === 'vod-hls-transcoding' && | ||
122 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
123 | } | ||
124 | |||
125 | function isRunnerJobVODAudioMergeUpdatePayloadValid ( | ||
126 | value: RunnerJobUpdatePayload, | ||
127 | type: RunnerJobType, | ||
128 | _files: UploadFilesForCheck | ||
129 | ) { | ||
130 | return type === 'vod-audio-merge-transcoding' && | ||
131 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
132 | } | ||
133 | |||
134 | function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( | ||
135 | value: RunnerJobUpdatePayload, | ||
136 | type: RunnerJobType, | ||
137 | files: UploadFilesForCheck | ||
138 | ) { | ||
139 | let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files | ||
140 | |||
141 | result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true }) | ||
142 | |||
143 | result &&= isFileValid({ | ||
144 | files, | ||
145 | field: 'payload[resolutionPlaylistFile]', | ||
146 | mimeTypeRegex: null, | ||
147 | maxSize: null, | ||
148 | optional: !value.resolutionPlaylistFilename | ||
149 | }) | ||
150 | |||
151 | if (files['payload[resolutionPlaylistFile]']) { | ||
152 | result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8') | ||
153 | } | ||
154 | |||
155 | return result && | ||
156 | isSafeFilename(value.videoChunkFilename, 'ts') && | ||
157 | ( | ||
158 | ( | ||
159 | value.type === 'remove-chunk' | ||
160 | ) || | ||
161 | ( | ||
162 | value.type === 'add-chunk' && | ||
163 | isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null }) | ||
164 | ) | ||
165 | ) | ||
166 | } | ||
diff --git a/server/helpers/custom-validators/runners/runners.ts b/server/helpers/custom-validators/runners/runners.ts new file mode 100644 index 000000000..953fac3b5 --- /dev/null +++ b/server/helpers/custom-validators/runners/runners.ts | |||
@@ -0,0 +1,30 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
3 | import { exists } from '../misc' | ||
4 | |||
5 | const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS | ||
6 | |||
7 | function isRunnerRegistrationTokenValid (value: string) { | ||
8 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) | ||
9 | } | ||
10 | |||
11 | function isRunnerTokenValid (value: string) { | ||
12 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) | ||
13 | } | ||
14 | |||
15 | function isRunnerNameValid (value: string) { | ||
16 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME) | ||
17 | } | ||
18 | |||
19 | function isRunnerDescriptionValid (value: string) { | ||
20 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION) | ||
21 | } | ||
22 | |||
23 | // --------------------------------------------------------------------------- | ||
24 | |||
25 | export { | ||
26 | isRunnerRegistrationTokenValid, | ||
27 | isRunnerTokenValid, | ||
28 | isRunnerNameValid, | ||
29 | isRunnerDescriptionValid | ||
30 | } | ||
diff --git a/server/helpers/debounce.ts b/server/helpers/debounce.ts new file mode 100644 index 000000000..77d99a894 --- /dev/null +++ b/server/helpers/debounce.ts | |||
@@ -0,0 +1,16 @@ | |||
1 | export function Debounce (config: { timeoutMS: number }) { | ||
2 | let timeoutRef: NodeJS.Timeout | ||
3 | |||
4 | return function (_target, _key, descriptor: PropertyDescriptor) { | ||
5 | const original = descriptor.value | ||
6 | |||
7 | descriptor.value = function (...args: any[]) { | ||
8 | clearTimeout(timeoutRef) | ||
9 | |||
10 | timeoutRef = setTimeout(() => { | ||
11 | original.apply(this, args) | ||
12 | |||
13 | }, config.timeoutMS) | ||
14 | } | ||
15 | } | ||
16 | } | ||
diff --git a/server/helpers/ffmpeg/codecs.ts b/server/helpers/ffmpeg/codecs.ts new file mode 100644 index 000000000..3bd7db396 --- /dev/null +++ b/server/helpers/ffmpeg/codecs.ts | |||
@@ -0,0 +1,64 @@ | |||
1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
2 | import { getAudioStream, getVideoStream } from '@shared/ffmpeg' | ||
3 | import { logger } from '../logger' | ||
4 | import { forceNumber } from '@shared/core-utils' | ||
5 | |||
6 | export async function getVideoStreamCodec (path: string) { | ||
7 | const videoStream = await getVideoStream(path) | ||
8 | if (!videoStream) return '' | ||
9 | |||
10 | const videoCodec = videoStream.codec_tag_string | ||
11 | |||
12 | if (videoCodec === 'vp09') return 'vp09.00.50.08' | ||
13 | if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' | ||
14 | |||
15 | const baseProfileMatrix = { | ||
16 | avc1: { | ||
17 | High: '6400', | ||
18 | Main: '4D40', | ||
19 | Baseline: '42E0' | ||
20 | }, | ||
21 | av01: { | ||
22 | High: '1', | ||
23 | Main: '0', | ||
24 | Professional: '2' | ||
25 | } | ||
26 | } | ||
27 | |||
28 | let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] | ||
29 | if (!baseProfile) { | ||
30 | logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) | ||
31 | baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback | ||
32 | } | ||
33 | |||
34 | if (videoCodec === 'av01') { | ||
35 | let level = videoStream.level.toString() | ||
36 | if (level.length === 1) level = `0${level}` | ||
37 | |||
38 | // Guess the tier indicator and bit depth | ||
39 | return `${videoCodec}.${baseProfile}.${level}M.08` | ||
40 | } | ||
41 | |||
42 | let level = forceNumber(videoStream.level).toString(16) | ||
43 | if (level.length === 1) level = `0${level}` | ||
44 | |||
45 | // Default, h264 codec | ||
46 | return `${videoCodec}.${baseProfile}${level}` | ||
47 | } | ||
48 | |||
49 | export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { | ||
50 | const { audioStream } = await getAudioStream(path, existingProbe) | ||
51 | |||
52 | if (!audioStream) return '' | ||
53 | |||
54 | const audioCodecName = audioStream.codec_name | ||
55 | |||
56 | if (audioCodecName === 'opus') return 'opus' | ||
57 | if (audioCodecName === 'vorbis') return 'vorbis' | ||
58 | if (audioCodecName === 'aac') return 'mp4a.40.2' | ||
59 | if (audioCodecName === 'mp3') return 'mp4a.40.34' | ||
60 | |||
61 | logger.warn('Cannot get audio codec of %s.', path, { audioStream }) | ||
62 | |||
63 | return 'mp4a.40.2' // Fallback | ||
64 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-commons.ts b/server/helpers/ffmpeg/ffmpeg-commons.ts deleted file mode 100644 index 3906a2089..000000000 --- a/server/helpers/ffmpeg/ffmpeg-commons.ts +++ /dev/null | |||
@@ -1,114 +0,0 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg' | ||
3 | import { execPromise } from '@server/helpers/core-utils' | ||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
5 | import { CONFIG } from '@server/initializers/config' | ||
6 | import { FFMPEG_NICE } from '@server/initializers/constants' | ||
7 | import { EncoderOptions } from '@shared/models' | ||
8 | |||
9 | const lTags = loggerTagsFactory('ffmpeg') | ||
10 | |||
11 | type StreamType = 'audio' | 'video' | ||
12 | |||
13 | function getFFmpeg (input: string, type: 'live' | 'vod') { | ||
14 | // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems | ||
15 | const command = ffmpeg(input, { | ||
16 | niceness: type === 'live' ? FFMPEG_NICE.LIVE : FFMPEG_NICE.VOD, | ||
17 | cwd: CONFIG.STORAGE.TMP_DIR | ||
18 | }) | ||
19 | |||
20 | const threads = type === 'live' | ||
21 | ? CONFIG.LIVE.TRANSCODING.THREADS | ||
22 | : CONFIG.TRANSCODING.THREADS | ||
23 | |||
24 | if (threads > 0) { | ||
25 | // If we don't set any threads ffmpeg will chose automatically | ||
26 | command.outputOption('-threads ' + threads) | ||
27 | } | ||
28 | |||
29 | return command | ||
30 | } | ||
31 | |||
32 | function getFFmpegVersion () { | ||
33 | return new Promise<string>((res, rej) => { | ||
34 | (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => { | ||
35 | if (err) return rej(err) | ||
36 | if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path')) | ||
37 | |||
38 | return execPromise(`${ffmpegPath} -version`) | ||
39 | .then(stdout => { | ||
40 | const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) | ||
41 | if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) | ||
42 | |||
43 | // Fix ffmpeg version that does not include patch version (4.4 for example) | ||
44 | let version = parsed[1] | ||
45 | if (version.match(/^\d+\.\d+$/)) { | ||
46 | version += '.0' | ||
47 | } | ||
48 | |||
49 | return res(version) | ||
50 | }) | ||
51 | .catch(err => rej(err)) | ||
52 | }) | ||
53 | }) | ||
54 | } | ||
55 | |||
56 | async function runCommand (options: { | ||
57 | command: FfmpegCommand | ||
58 | silent?: boolean // false by default | ||
59 | job?: Job | ||
60 | }) { | ||
61 | const { command, silent = false, job } = options | ||
62 | |||
63 | return new Promise<void>((res, rej) => { | ||
64 | let shellCommand: string | ||
65 | |||
66 | command.on('start', cmdline => { shellCommand = cmdline }) | ||
67 | |||
68 | command.on('error', (err, stdout, stderr) => { | ||
69 | if (silent !== true) logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...lTags() }) | ||
70 | |||
71 | rej(err) | ||
72 | }) | ||
73 | |||
74 | command.on('end', (stdout, stderr) => { | ||
75 | logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...lTags() }) | ||
76 | |||
77 | res() | ||
78 | }) | ||
79 | |||
80 | if (job) { | ||
81 | command.on('progress', progress => { | ||
82 | if (!progress.percent) return | ||
83 | |||
84 | job.updateProgress(Math.round(progress.percent)) | ||
85 | .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err, ...lTags() })) | ||
86 | }) | ||
87 | } | ||
88 | |||
89 | command.run() | ||
90 | }) | ||
91 | } | ||
92 | |||
93 | function buildStreamSuffix (base: string, streamNum?: number) { | ||
94 | if (streamNum !== undefined) { | ||
95 | return `${base}:${streamNum}` | ||
96 | } | ||
97 | |||
98 | return base | ||
99 | } | ||
100 | |||
101 | function getScaleFilter (options: EncoderOptions): string { | ||
102 | if (options.scaleFilter) return options.scaleFilter.name | ||
103 | |||
104 | return 'scale' | ||
105 | } | ||
106 | |||
107 | export { | ||
108 | getFFmpeg, | ||
109 | getFFmpegVersion, | ||
110 | runCommand, | ||
111 | StreamType, | ||
112 | buildStreamSuffix, | ||
113 | getScaleFilter | ||
114 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-edition.ts b/server/helpers/ffmpeg/ffmpeg-edition.ts deleted file mode 100644 index 02c5ea8de..000000000 --- a/server/helpers/ffmpeg/ffmpeg-edition.ts +++ /dev/null | |||
@@ -1,258 +0,0 @@ | |||
1 | import { FilterSpecification } from 'fluent-ffmpeg' | ||
2 | import { VIDEO_FILTERS } from '@server/initializers/constants' | ||
3 | import { AvailableEncoders } from '@shared/models' | ||
4 | import { logger, loggerTagsFactory } from '../logger' | ||
5 | import { getFFmpeg, runCommand } from './ffmpeg-commons' | ||
6 | import { presetVOD } from './ffmpeg-presets' | ||
7 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe-utils' | ||
8 | |||
9 | const lTags = loggerTagsFactory('ffmpeg') | ||
10 | |||
11 | async function cutVideo (options: { | ||
12 | inputPath: string | ||
13 | outputPath: string | ||
14 | start?: number | ||
15 | end?: number | ||
16 | |||
17 | availableEncoders: AvailableEncoders | ||
18 | profile: string | ||
19 | }) { | ||
20 | const { inputPath, outputPath, availableEncoders, profile } = options | ||
21 | |||
22 | logger.debug('Will cut the video.', { options, ...lTags() }) | ||
23 | |||
24 | const mainProbe = await ffprobePromise(inputPath) | ||
25 | const fps = await getVideoStreamFPS(inputPath, mainProbe) | ||
26 | const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) | ||
27 | |||
28 | let command = getFFmpeg(inputPath, 'vod') | ||
29 | .output(outputPath) | ||
30 | |||
31 | command = await presetVOD({ | ||
32 | command, | ||
33 | input: inputPath, | ||
34 | availableEncoders, | ||
35 | profile, | ||
36 | resolution, | ||
37 | fps, | ||
38 | canCopyAudio: false, | ||
39 | canCopyVideo: false | ||
40 | }) | ||
41 | |||
42 | if (options.start) { | ||
43 | command.outputOption('-ss ' + options.start) | ||
44 | } | ||
45 | |||
46 | if (options.end) { | ||
47 | command.outputOption('-to ' + options.end) | ||
48 | } | ||
49 | |||
50 | await runCommand({ command }) | ||
51 | } | ||
52 | |||
53 | async function addWatermark (options: { | ||
54 | inputPath: string | ||
55 | watermarkPath: string | ||
56 | outputPath: string | ||
57 | |||
58 | availableEncoders: AvailableEncoders | ||
59 | profile: string | ||
60 | }) { | ||
61 | const { watermarkPath, inputPath, outputPath, availableEncoders, profile } = options | ||
62 | |||
63 | logger.debug('Will add watermark to the video.', { options, ...lTags() }) | ||
64 | |||
65 | const videoProbe = await ffprobePromise(inputPath) | ||
66 | const fps = await getVideoStreamFPS(inputPath, videoProbe) | ||
67 | const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) | ||
68 | |||
69 | let command = getFFmpeg(inputPath, 'vod') | ||
70 | .output(outputPath) | ||
71 | command.input(watermarkPath) | ||
72 | |||
73 | command = await presetVOD({ | ||
74 | command, | ||
75 | input: inputPath, | ||
76 | availableEncoders, | ||
77 | profile, | ||
78 | resolution, | ||
79 | fps, | ||
80 | canCopyAudio: true, | ||
81 | canCopyVideo: false | ||
82 | }) | ||
83 | |||
84 | const complexFilter: FilterSpecification[] = [ | ||
85 | // Scale watermark | ||
86 | { | ||
87 | inputs: [ '[1]', '[0]' ], | ||
88 | filter: 'scale2ref', | ||
89 | options: { | ||
90 | w: 'oh*mdar', | ||
91 | h: `ih*${VIDEO_FILTERS.WATERMARK.SIZE_RATIO}` | ||
92 | }, | ||
93 | outputs: [ '[watermark]', '[video]' ] | ||
94 | }, | ||
95 | |||
96 | { | ||
97 | inputs: [ '[video]', '[watermark]' ], | ||
98 | filter: 'overlay', | ||
99 | options: { | ||
100 | x: `main_w - overlay_w - (main_h * ${VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO})`, | ||
101 | y: `main_h * ${VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO}` | ||
102 | } | ||
103 | } | ||
104 | ] | ||
105 | |||
106 | command.complexFilter(complexFilter) | ||
107 | |||
108 | await runCommand({ command }) | ||
109 | } | ||
110 | |||
111 | async function addIntroOutro (options: { | ||
112 | inputPath: string | ||
113 | introOutroPath: string | ||
114 | outputPath: string | ||
115 | type: 'intro' | 'outro' | ||
116 | |||
117 | availableEncoders: AvailableEncoders | ||
118 | profile: string | ||
119 | }) { | ||
120 | const { introOutroPath, inputPath, outputPath, availableEncoders, profile, type } = options | ||
121 | |||
122 | logger.debug('Will add intro/outro to the video.', { options, ...lTags() }) | ||
123 | |||
124 | const mainProbe = await ffprobePromise(inputPath) | ||
125 | const fps = await getVideoStreamFPS(inputPath, mainProbe) | ||
126 | const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) | ||
127 | const mainHasAudio = await hasAudioStream(inputPath, mainProbe) | ||
128 | |||
129 | const introOutroProbe = await ffprobePromise(introOutroPath) | ||
130 | const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) | ||
131 | |||
132 | let command = getFFmpeg(inputPath, 'vod') | ||
133 | .output(outputPath) | ||
134 | |||
135 | command.input(introOutroPath) | ||
136 | |||
137 | if (!introOutroHasAudio && mainHasAudio) { | ||
138 | const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) | ||
139 | |||
140 | command.input('anullsrc') | ||
141 | command.withInputFormat('lavfi') | ||
142 | command.withInputOption('-t ' + duration) | ||
143 | } | ||
144 | |||
145 | command = await presetVOD({ | ||
146 | command, | ||
147 | input: inputPath, | ||
148 | availableEncoders, | ||
149 | profile, | ||
150 | resolution, | ||
151 | fps, | ||
152 | canCopyAudio: false, | ||
153 | canCopyVideo: false | ||
154 | }) | ||
155 | |||
156 | // Add black background to correctly scale intro/outro with padding | ||
157 | const complexFilter: FilterSpecification[] = [ | ||
158 | { | ||
159 | inputs: [ '1', '0' ], | ||
160 | filter: 'scale2ref', | ||
161 | options: { | ||
162 | w: 'iw', | ||
163 | h: `ih` | ||
164 | }, | ||
165 | outputs: [ 'intro-outro', 'main' ] | ||
166 | }, | ||
167 | { | ||
168 | inputs: [ 'intro-outro', 'main' ], | ||
169 | filter: 'scale2ref', | ||
170 | options: { | ||
171 | w: 'iw', | ||
172 | h: `ih` | ||
173 | }, | ||
174 | outputs: [ 'to-scale', 'main' ] | ||
175 | }, | ||
176 | { | ||
177 | inputs: 'to-scale', | ||
178 | filter: 'drawbox', | ||
179 | options: { | ||
180 | t: 'fill' | ||
181 | }, | ||
182 | outputs: [ 'to-scale-bg' ] | ||
183 | }, | ||
184 | { | ||
185 | inputs: [ '1', 'to-scale-bg' ], | ||
186 | filter: 'scale2ref', | ||
187 | options: { | ||
188 | w: 'iw', | ||
189 | h: 'ih', | ||
190 | force_original_aspect_ratio: 'decrease', | ||
191 | flags: 'spline' | ||
192 | }, | ||
193 | outputs: [ 'to-scale', 'to-scale-bg' ] | ||
194 | }, | ||
195 | { | ||
196 | inputs: [ 'to-scale-bg', 'to-scale' ], | ||
197 | filter: 'overlay', | ||
198 | options: { | ||
199 | x: '(main_w - overlay_w)/2', | ||
200 | y: '(main_h - overlay_h)/2' | ||
201 | }, | ||
202 | outputs: 'intro-outro-resized' | ||
203 | } | ||
204 | ] | ||
205 | |||
206 | const concatFilter = { | ||
207 | inputs: [], | ||
208 | filter: 'concat', | ||
209 | options: { | ||
210 | n: 2, | ||
211 | v: 1, | ||
212 | unsafe: 1 | ||
213 | }, | ||
214 | outputs: [ 'v' ] | ||
215 | } | ||
216 | |||
217 | const introOutroFilterInputs = [ 'intro-outro-resized' ] | ||
218 | const mainFilterInputs = [ 'main' ] | ||
219 | |||
220 | if (mainHasAudio) { | ||
221 | mainFilterInputs.push('0:a') | ||
222 | |||
223 | if (introOutroHasAudio) { | ||
224 | introOutroFilterInputs.push('1:a') | ||
225 | } else { | ||
226 | // Silent input | ||
227 | introOutroFilterInputs.push('2:a') | ||
228 | } | ||
229 | } | ||
230 | |||
231 | if (type === 'intro') { | ||
232 | concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] | ||
233 | } else { | ||
234 | concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] | ||
235 | } | ||
236 | |||
237 | if (mainHasAudio) { | ||
238 | concatFilter.options['a'] = 1 | ||
239 | concatFilter.outputs.push('a') | ||
240 | |||
241 | command.outputOption('-map [a]') | ||
242 | } | ||
243 | |||
244 | command.outputOption('-map [v]') | ||
245 | |||
246 | complexFilter.push(concatFilter) | ||
247 | command.complexFilter(complexFilter) | ||
248 | |||
249 | await runCommand({ command }) | ||
250 | } | ||
251 | |||
252 | // --------------------------------------------------------------------------- | ||
253 | |||
254 | export { | ||
255 | cutVideo, | ||
256 | addIntroOutro, | ||
257 | addWatermark | ||
258 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-encoders.ts b/server/helpers/ffmpeg/ffmpeg-encoders.ts deleted file mode 100644 index 5bd80ba05..000000000 --- a/server/helpers/ffmpeg/ffmpeg-encoders.ts +++ /dev/null | |||
@@ -1,116 +0,0 @@ | |||
1 | import { getAvailableEncoders } from 'fluent-ffmpeg' | ||
2 | import { pick } from '@shared/core-utils' | ||
3 | import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models' | ||
4 | import { promisify0 } from '../core-utils' | ||
5 | import { logger, loggerTagsFactory } from '../logger' | ||
6 | |||
7 | const lTags = loggerTagsFactory('ffmpeg') | ||
8 | |||
9 | // Detect supported encoders by ffmpeg | ||
10 | let supportedEncoders: Map<string, boolean> | ||
11 | async function checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise<Map<string, boolean>> { | ||
12 | if (supportedEncoders !== undefined) { | ||
13 | return supportedEncoders | ||
14 | } | ||
15 | |||
16 | const getAvailableEncodersPromise = promisify0(getAvailableEncoders) | ||
17 | const availableFFmpegEncoders = await getAvailableEncodersPromise() | ||
18 | |||
19 | const searchEncoders = new Set<string>() | ||
20 | for (const type of [ 'live', 'vod' ]) { | ||
21 | for (const streamType of [ 'audio', 'video' ]) { | ||
22 | for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { | ||
23 | searchEncoders.add(encoder) | ||
24 | } | ||
25 | } | ||
26 | } | ||
27 | |||
28 | supportedEncoders = new Map<string, boolean>() | ||
29 | |||
30 | for (const searchEncoder of searchEncoders) { | ||
31 | supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) | ||
32 | } | ||
33 | |||
34 | logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...lTags() }) | ||
35 | |||
36 | return supportedEncoders | ||
37 | } | ||
38 | |||
39 | function resetSupportedEncoders () { | ||
40 | supportedEncoders = undefined | ||
41 | } | ||
42 | |||
43 | // Run encoder builder depending on available encoders | ||
44 | // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one | ||
45 | // If the default one does not exist, check the next encoder | ||
46 | async function getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { | ||
47 | streamType: 'video' | 'audio' | ||
48 | input: string | ||
49 | |||
50 | availableEncoders: AvailableEncoders | ||
51 | profile: string | ||
52 | |||
53 | videoType: 'vod' | 'live' | ||
54 | }) { | ||
55 | const { availableEncoders, profile, streamType, videoType } = options | ||
56 | |||
57 | const encodersToTry = availableEncoders.encodersToTry[videoType][streamType] | ||
58 | const encoders = availableEncoders.available[videoType] | ||
59 | |||
60 | for (const encoder of encodersToTry) { | ||
61 | if (!(await checkFFmpegEncoders(availableEncoders)).get(encoder)) { | ||
62 | logger.debug('Encoder %s not available in ffmpeg, skipping.', encoder, lTags()) | ||
63 | continue | ||
64 | } | ||
65 | |||
66 | if (!encoders[encoder]) { | ||
67 | logger.debug('Encoder %s not available in peertube encoders, skipping.', encoder, lTags()) | ||
68 | continue | ||
69 | } | ||
70 | |||
71 | // An object containing available profiles for this encoder | ||
72 | const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder] | ||
73 | let builder = builderProfiles[profile] | ||
74 | |||
75 | if (!builder) { | ||
76 | logger.debug('Profile %s for encoder %s not available. Fallback to default.', profile, encoder, lTags()) | ||
77 | builder = builderProfiles.default | ||
78 | |||
79 | if (!builder) { | ||
80 | logger.debug('Default profile for encoder %s not available. Try next available encoder.', encoder, lTags()) | ||
81 | continue | ||
82 | } | ||
83 | } | ||
84 | |||
85 | const result = await builder( | ||
86 | pick(options, [ | ||
87 | 'input', | ||
88 | 'canCopyAudio', | ||
89 | 'canCopyVideo', | ||
90 | 'resolution', | ||
91 | 'inputBitrate', | ||
92 | 'fps', | ||
93 | 'inputRatio', | ||
94 | 'streamNum' | ||
95 | ]) | ||
96 | ) | ||
97 | |||
98 | return { | ||
99 | result, | ||
100 | |||
101 | // If we don't have output options, then copy the input stream | ||
102 | encoder: result.copy === true | ||
103 | ? 'copy' | ||
104 | : encoder | ||
105 | } | ||
106 | } | ||
107 | |||
108 | return null | ||
109 | } | ||
110 | |||
111 | export { | ||
112 | checkFFmpegEncoders, | ||
113 | resetSupportedEncoders, | ||
114 | |||
115 | getEncoderBuilderResult | ||
116 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-image.ts b/server/helpers/ffmpeg/ffmpeg-image.ts new file mode 100644 index 000000000..0bb0ff2c0 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-image.ts | |||
@@ -0,0 +1,14 @@ | |||
1 | import { FFmpegImage } from '@shared/ffmpeg' | ||
2 | import { getFFmpegCommandWrapperOptions } from './ffmpeg-options' | ||
3 | |||
4 | export function processGIF (options: Parameters<FFmpegImage['processGIF']>[0]) { | ||
5 | return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options) | ||
6 | } | ||
7 | |||
8 | export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) { | ||
9 | return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options) | ||
10 | } | ||
11 | |||
12 | export function convertWebPToJPG (options: Parameters<FFmpegImage['convertWebPToJPG']>[0]) { | ||
13 | return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options) | ||
14 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-images.ts b/server/helpers/ffmpeg/ffmpeg-images.ts deleted file mode 100644 index 7f64c6d0a..000000000 --- a/server/helpers/ffmpeg/ffmpeg-images.ts +++ /dev/null | |||
@@ -1,46 +0,0 @@ | |||
1 | import ffmpeg from 'fluent-ffmpeg' | ||
2 | import { FFMPEG_NICE } from '@server/initializers/constants' | ||
3 | import { runCommand } from './ffmpeg-commons' | ||
4 | |||
5 | function convertWebPToJPG (path: string, destination: string): Promise<void> { | ||
6 | const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL }) | ||
7 | .output(destination) | ||
8 | |||
9 | return runCommand({ command, silent: true }) | ||
10 | } | ||
11 | |||
12 | function processGIF ( | ||
13 | path: string, | ||
14 | destination: string, | ||
15 | newSize: { width: number, height: number } | ||
16 | ): Promise<void> { | ||
17 | const command = ffmpeg(path, { niceness: FFMPEG_NICE.THUMBNAIL }) | ||
18 | .fps(20) | ||
19 | .size(`${newSize.width}x${newSize.height}`) | ||
20 | .output(destination) | ||
21 | |||
22 | return runCommand({ command }) | ||
23 | } | ||
24 | |||
25 | async function generateThumbnailFromVideo (fromPath: string, folder: string, imageName: string) { | ||
26 | const pendingImageName = 'pending-' + imageName | ||
27 | |||
28 | const options = { | ||
29 | filename: pendingImageName, | ||
30 | count: 1, | ||
31 | folder | ||
32 | } | ||
33 | |||
34 | return new Promise<string>((res, rej) => { | ||
35 | ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL }) | ||
36 | .on('error', rej) | ||
37 | .on('end', () => res(imageName)) | ||
38 | .thumbnail(options) | ||
39 | }) | ||
40 | } | ||
41 | |||
42 | export { | ||
43 | convertWebPToJPG, | ||
44 | processGIF, | ||
45 | generateThumbnailFromVideo | ||
46 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-live.ts b/server/helpers/ffmpeg/ffmpeg-live.ts deleted file mode 100644 index 379d7b1ad..000000000 --- a/server/helpers/ffmpeg/ffmpeg-live.ts +++ /dev/null | |||
@@ -1,204 +0,0 @@ | |||
1 | import { FfmpegCommand, FilterSpecification } from 'fluent-ffmpeg' | ||
2 | import { join } from 'path' | ||
3 | import { VIDEO_LIVE } from '@server/initializers/constants' | ||
4 | import { AvailableEncoders, LiveVideoLatencyMode } from '@shared/models' | ||
5 | import { logger, loggerTagsFactory } from '../logger' | ||
6 | import { buildStreamSuffix, getFFmpeg, getScaleFilter, StreamType } from './ffmpeg-commons' | ||
7 | import { getEncoderBuilderResult } from './ffmpeg-encoders' | ||
8 | import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './ffmpeg-presets' | ||
9 | import { computeFPS } from './ffprobe-utils' | ||
10 | |||
11 | const lTags = loggerTagsFactory('ffmpeg') | ||
12 | |||
13 | async function getLiveTranscodingCommand (options: { | ||
14 | inputUrl: string | ||
15 | |||
16 | outPath: string | ||
17 | masterPlaylistName: string | ||
18 | latencyMode: LiveVideoLatencyMode | ||
19 | |||
20 | resolutions: number[] | ||
21 | |||
22 | // Input information | ||
23 | fps: number | ||
24 | bitrate: number | ||
25 | ratio: number | ||
26 | hasAudio: boolean | ||
27 | |||
28 | availableEncoders: AvailableEncoders | ||
29 | profile: string | ||
30 | }) { | ||
31 | const { | ||
32 | inputUrl, | ||
33 | outPath, | ||
34 | resolutions, | ||
35 | fps, | ||
36 | bitrate, | ||
37 | availableEncoders, | ||
38 | profile, | ||
39 | masterPlaylistName, | ||
40 | ratio, | ||
41 | latencyMode, | ||
42 | hasAudio | ||
43 | } = options | ||
44 | |||
45 | const command = getFFmpeg(inputUrl, 'live') | ||
46 | |||
47 | const varStreamMap: string[] = [] | ||
48 | |||
49 | const complexFilter: FilterSpecification[] = [ | ||
50 | { | ||
51 | inputs: '[v:0]', | ||
52 | filter: 'split', | ||
53 | options: resolutions.length, | ||
54 | outputs: resolutions.map(r => `vtemp${r}`) | ||
55 | } | ||
56 | ] | ||
57 | |||
58 | command.outputOption('-sc_threshold 0') | ||
59 | |||
60 | addDefaultEncoderGlobalParams(command) | ||
61 | |||
62 | for (let i = 0; i < resolutions.length; i++) { | ||
63 | const streamMap: string[] = [] | ||
64 | const resolution = resolutions[i] | ||
65 | const resolutionFPS = computeFPS(fps, resolution) | ||
66 | |||
67 | const baseEncoderBuilderParams = { | ||
68 | input: inputUrl, | ||
69 | |||
70 | availableEncoders, | ||
71 | profile, | ||
72 | |||
73 | canCopyAudio: true, | ||
74 | canCopyVideo: true, | ||
75 | |||
76 | inputBitrate: bitrate, | ||
77 | inputRatio: ratio, | ||
78 | |||
79 | resolution, | ||
80 | fps: resolutionFPS, | ||
81 | |||
82 | streamNum: i, | ||
83 | videoType: 'live' as 'live' | ||
84 | } | ||
85 | |||
86 | { | ||
87 | const streamType: StreamType = 'video' | ||
88 | const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) | ||
89 | if (!builderResult) { | ||
90 | throw new Error('No available live video encoder found') | ||
91 | } | ||
92 | |||
93 | command.outputOption(`-map [vout${resolution}]`) | ||
94 | |||
95 | addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i }) | ||
96 | |||
97 | logger.debug( | ||
98 | 'Apply ffmpeg live video params from %s using %s profile.', builderResult.encoder, profile, | ||
99 | { builderResult, fps: resolutionFPS, resolution, ...lTags() } | ||
100 | ) | ||
101 | |||
102 | command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) | ||
103 | applyEncoderOptions(command, builderResult.result) | ||
104 | |||
105 | complexFilter.push({ | ||
106 | inputs: `vtemp${resolution}`, | ||
107 | filter: getScaleFilter(builderResult.result), | ||
108 | options: `w=-2:h=${resolution}`, | ||
109 | outputs: `vout${resolution}` | ||
110 | }) | ||
111 | |||
112 | streamMap.push(`v:${i}`) | ||
113 | } | ||
114 | |||
115 | if (hasAudio) { | ||
116 | const streamType: StreamType = 'audio' | ||
117 | const builderResult = await getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) | ||
118 | if (!builderResult) { | ||
119 | throw new Error('No available live audio encoder found') | ||
120 | } | ||
121 | |||
122 | command.outputOption('-map a:0') | ||
123 | |||
124 | addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps: resolutionFPS, streamNum: i }) | ||
125 | |||
126 | logger.debug( | ||
127 | 'Apply ffmpeg live audio params from %s using %s profile.', builderResult.encoder, profile, | ||
128 | { builderResult, fps: resolutionFPS, resolution, ...lTags() } | ||
129 | ) | ||
130 | |||
131 | command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) | ||
132 | applyEncoderOptions(command, builderResult.result) | ||
133 | |||
134 | streamMap.push(`a:${i}`) | ||
135 | } | ||
136 | |||
137 | varStreamMap.push(streamMap.join(',')) | ||
138 | } | ||
139 | |||
140 | command.complexFilter(complexFilter) | ||
141 | |||
142 | addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode }) | ||
143 | |||
144 | command.outputOption('-var_stream_map', varStreamMap.join(' ')) | ||
145 | |||
146 | return command | ||
147 | } | ||
148 | |||
149 | function getLiveMuxingCommand (options: { | ||
150 | inputUrl: string | ||
151 | outPath: string | ||
152 | masterPlaylistName: string | ||
153 | latencyMode: LiveVideoLatencyMode | ||
154 | }) { | ||
155 | const { inputUrl, outPath, masterPlaylistName, latencyMode } = options | ||
156 | |||
157 | const command = getFFmpeg(inputUrl, 'live') | ||
158 | |||
159 | command.outputOption('-c:v copy') | ||
160 | command.outputOption('-c:a copy') | ||
161 | command.outputOption('-map 0:a?') | ||
162 | command.outputOption('-map 0:v?') | ||
163 | |||
164 | addDefaultLiveHLSParams({ command, outPath, masterPlaylistName, latencyMode }) | ||
165 | |||
166 | return command | ||
167 | } | ||
168 | |||
169 | function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { | ||
170 | if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { | ||
171 | return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY | ||
172 | } | ||
173 | |||
174 | return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY | ||
175 | } | ||
176 | |||
177 | // --------------------------------------------------------------------------- | ||
178 | |||
179 | export { | ||
180 | getLiveSegmentTime, | ||
181 | |||
182 | getLiveTranscodingCommand, | ||
183 | getLiveMuxingCommand | ||
184 | } | ||
185 | |||
186 | // --------------------------------------------------------------------------- | ||
187 | |||
188 | function addDefaultLiveHLSParams (options: { | ||
189 | command: FfmpegCommand | ||
190 | outPath: string | ||
191 | masterPlaylistName: string | ||
192 | latencyMode: LiveVideoLatencyMode | ||
193 | }) { | ||
194 | const { command, outPath, masterPlaylistName, latencyMode } = options | ||
195 | |||
196 | command.outputOption('-hls_time ' + getLiveSegmentTime(latencyMode)) | ||
197 | command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) | ||
198 | command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time') | ||
199 | command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) | ||
200 | command.outputOption('-master_pl_name ' + masterPlaylistName) | ||
201 | command.outputOption(`-f hls`) | ||
202 | |||
203 | command.output(join(outPath, '%v.m3u8')) | ||
204 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-options.ts b/server/helpers/ffmpeg/ffmpeg-options.ts new file mode 100644 index 000000000..db6350d39 --- /dev/null +++ b/server/helpers/ffmpeg/ffmpeg-options.ts | |||
@@ -0,0 +1,45 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { FFMPEG_NICE } from '@server/initializers/constants' | ||
4 | import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg' | ||
5 | import { AvailableEncoders } from '@shared/models' | ||
6 | |||
7 | type CommandType = 'live' | 'vod' | 'thumbnail' | ||
8 | |||
9 | export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions { | ||
10 | return { | ||
11 | availableEncoders, | ||
12 | profile: getProfile(type), | ||
13 | |||
14 | niceness: FFMPEG_NICE[type], | ||
15 | tmpDirectory: CONFIG.STORAGE.TMP_DIR, | ||
16 | threads: getThreads(type), | ||
17 | |||
18 | logger: { | ||
19 | debug: logger.debug.bind(logger), | ||
20 | info: logger.info.bind(logger), | ||
21 | warn: logger.warn.bind(logger), | ||
22 | error: logger.error.bind(logger) | ||
23 | }, | ||
24 | lTags: { tags: [ 'ffmpeg' ] } | ||
25 | } | ||
26 | } | ||
27 | |||
28 | // --------------------------------------------------------------------------- | ||
29 | // Private | ||
30 | // --------------------------------------------------------------------------- | ||
31 | |||
32 | function getThreads (type: CommandType) { | ||
33 | if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS | ||
34 | if (type === 'vod') return CONFIG.TRANSCODING.THREADS | ||
35 | |||
36 | // Auto | ||
37 | return 0 | ||
38 | } | ||
39 | |||
40 | function getProfile (type: CommandType) { | ||
41 | if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE | ||
42 | if (type === 'vod') return CONFIG.TRANSCODING.PROFILE | ||
43 | |||
44 | return undefined | ||
45 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-presets.ts b/server/helpers/ffmpeg/ffmpeg-presets.ts deleted file mode 100644 index d1160a4a2..000000000 --- a/server/helpers/ffmpeg/ffmpeg-presets.ts +++ /dev/null | |||
@@ -1,156 +0,0 @@ | |||
1 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { pick } from '@shared/core-utils' | ||
4 | import { AvailableEncoders, EncoderOptions } from '@shared/models' | ||
5 | import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-commons' | ||
6 | import { getEncoderBuilderResult } from './ffmpeg-encoders' | ||
7 | import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from './ffprobe-utils' | ||
8 | |||
9 | const lTags = loggerTagsFactory('ffmpeg') | ||
10 | |||
11 | // --------------------------------------------------------------------------- | ||
12 | |||
13 | function addDefaultEncoderGlobalParams (command: FfmpegCommand) { | ||
14 | // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 | ||
15 | command.outputOption('-max_muxing_queue_size 1024') | ||
16 | // strip all metadata | ||
17 | .outputOption('-map_metadata -1') | ||
18 | // allows import of source material with incompatible pixel formats (e.g. MJPEG video) | ||
19 | .outputOption('-pix_fmt yuv420p') | ||
20 | } | ||
21 | |||
22 | function addDefaultEncoderParams (options: { | ||
23 | command: FfmpegCommand | ||
24 | encoder: 'libx264' | string | ||
25 | fps: number | ||
26 | |||
27 | streamNum?: number | ||
28 | }) { | ||
29 | const { command, encoder, fps, streamNum } = options | ||
30 | |||
31 | if (encoder === 'libx264') { | ||
32 | // 3.1 is the minimal resource allocation for our highest supported resolution | ||
33 | command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') | ||
34 | |||
35 | if (fps) { | ||
36 | // Keyframe interval of 2 seconds for faster seeking and resolution switching. | ||
37 | // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html | ||
38 | // https://superuser.com/a/908325 | ||
39 | command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) | ||
40 | } | ||
41 | } | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | async function presetVOD (options: { | ||
47 | command: FfmpegCommand | ||
48 | input: string | ||
49 | |||
50 | availableEncoders: AvailableEncoders | ||
51 | profile: string | ||
52 | |||
53 | canCopyAudio: boolean | ||
54 | canCopyVideo: boolean | ||
55 | |||
56 | resolution: number | ||
57 | fps: number | ||
58 | |||
59 | scaleFilterValue?: string | ||
60 | }) { | ||
61 | const { command, input, profile, resolution, fps, scaleFilterValue } = options | ||
62 | |||
63 | let localCommand = command | ||
64 | .format('mp4') | ||
65 | .outputOption('-movflags faststart') | ||
66 | |||
67 | addDefaultEncoderGlobalParams(command) | ||
68 | |||
69 | const probe = await ffprobePromise(input) | ||
70 | |||
71 | // Audio encoder | ||
72 | const bitrate = await getVideoStreamBitrate(input, probe) | ||
73 | const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) | ||
74 | |||
75 | let streamsToProcess: StreamType[] = [ 'audio', 'video' ] | ||
76 | |||
77 | if (!await hasAudioStream(input, probe)) { | ||
78 | localCommand = localCommand.noAudio() | ||
79 | streamsToProcess = [ 'video' ] | ||
80 | } | ||
81 | |||
82 | for (const streamType of streamsToProcess) { | ||
83 | const builderResult = await getEncoderBuilderResult({ | ||
84 | ...pick(options, [ 'availableEncoders', 'canCopyAudio', 'canCopyVideo' ]), | ||
85 | |||
86 | input, | ||
87 | inputBitrate: bitrate, | ||
88 | inputRatio: videoStreamDimensions?.ratio || 0, | ||
89 | |||
90 | profile, | ||
91 | resolution, | ||
92 | fps, | ||
93 | streamType, | ||
94 | |||
95 | videoType: 'vod' as 'vod' | ||
96 | }) | ||
97 | |||
98 | if (!builderResult) { | ||
99 | throw new Error('No available encoder found for stream ' + streamType) | ||
100 | } | ||
101 | |||
102 | logger.debug( | ||
103 | 'Apply ffmpeg params from %s for %s stream of input %s using %s profile.', | ||
104 | builderResult.encoder, streamType, input, profile, | ||
105 | { builderResult, resolution, fps, ...lTags() } | ||
106 | ) | ||
107 | |||
108 | if (streamType === 'video') { | ||
109 | localCommand.videoCodec(builderResult.encoder) | ||
110 | |||
111 | if (scaleFilterValue) { | ||
112 | localCommand.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) | ||
113 | } | ||
114 | } else if (streamType === 'audio') { | ||
115 | localCommand.audioCodec(builderResult.encoder) | ||
116 | } | ||
117 | |||
118 | applyEncoderOptions(localCommand, builderResult.result) | ||
119 | addDefaultEncoderParams({ command: localCommand, encoder: builderResult.encoder, fps }) | ||
120 | } | ||
121 | |||
122 | return localCommand | ||
123 | } | ||
124 | |||
125 | function presetCopy (command: FfmpegCommand): FfmpegCommand { | ||
126 | return command | ||
127 | .format('mp4') | ||
128 | .videoCodec('copy') | ||
129 | .audioCodec('copy') | ||
130 | } | ||
131 | |||
132 | function presetOnlyAudio (command: FfmpegCommand): FfmpegCommand { | ||
133 | return command | ||
134 | .format('mp4') | ||
135 | .audioCodec('copy') | ||
136 | .noVideo() | ||
137 | } | ||
138 | |||
139 | function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions): FfmpegCommand { | ||
140 | return command | ||
141 | .inputOptions(options.inputOptions ?? []) | ||
142 | .outputOptions(options.outputOptions ?? []) | ||
143 | } | ||
144 | |||
145 | // --------------------------------------------------------------------------- | ||
146 | |||
147 | export { | ||
148 | presetVOD, | ||
149 | presetCopy, | ||
150 | presetOnlyAudio, | ||
151 | |||
152 | addDefaultEncoderGlobalParams, | ||
153 | addDefaultEncoderParams, | ||
154 | |||
155 | applyEncoderOptions | ||
156 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-vod.ts b/server/helpers/ffmpeg/ffmpeg-vod.ts deleted file mode 100644 index d84703eb9..000000000 --- a/server/helpers/ffmpeg/ffmpeg-vod.ts +++ /dev/null | |||
@@ -1,267 +0,0 @@ | |||
1 | import { MutexInterface } from 'async-mutex' | ||
2 | import { Job } from 'bullmq' | ||
3 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
4 | import { readFile, writeFile } from 'fs-extra' | ||
5 | import { dirname } from 'path' | ||
6 | import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
7 | import { pick } from '@shared/core-utils' | ||
8 | import { AvailableEncoders, VideoResolution } from '@shared/models' | ||
9 | import { logger, loggerTagsFactory } from '../logger' | ||
10 | import { getFFmpeg, runCommand } from './ffmpeg-commons' | ||
11 | import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets' | ||
12 | import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils' | ||
13 | |||
14 | const lTags = loggerTagsFactory('ffmpeg') | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' | ||
19 | |||
20 | interface BaseTranscodeVODOptions { | ||
21 | type: TranscodeVODOptionsType | ||
22 | |||
23 | inputPath: string | ||
24 | outputPath: string | ||
25 | |||
26 | // Will be released after the ffmpeg started | ||
27 | // To prevent a bug where the input file does not exist anymore when running ffmpeg | ||
28 | inputFileMutexReleaser: MutexInterface.Releaser | ||
29 | |||
30 | availableEncoders: AvailableEncoders | ||
31 | profile: string | ||
32 | |||
33 | resolution: number | ||
34 | |||
35 | job?: Job | ||
36 | } | ||
37 | |||
38 | interface HLSTranscodeOptions extends BaseTranscodeVODOptions { | ||
39 | type: 'hls' | ||
40 | copyCodecs: boolean | ||
41 | hlsPlaylist: { | ||
42 | videoFilename: string | ||
43 | } | ||
44 | } | ||
45 | |||
46 | interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { | ||
47 | type: 'hls-from-ts' | ||
48 | |||
49 | isAAC: boolean | ||
50 | |||
51 | hlsPlaylist: { | ||
52 | videoFilename: string | ||
53 | } | ||
54 | } | ||
55 | |||
56 | interface QuickTranscodeOptions extends BaseTranscodeVODOptions { | ||
57 | type: 'quick-transcode' | ||
58 | } | ||
59 | |||
60 | interface VideoTranscodeOptions extends BaseTranscodeVODOptions { | ||
61 | type: 'video' | ||
62 | } | ||
63 | |||
64 | interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { | ||
65 | type: 'merge-audio' | ||
66 | audioPath: string | ||
67 | } | ||
68 | |||
69 | interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { | ||
70 | type: 'only-audio' | ||
71 | } | ||
72 | |||
73 | type TranscodeVODOptions = | ||
74 | HLSTranscodeOptions | ||
75 | | HLSFromTSTranscodeOptions | ||
76 | | VideoTranscodeOptions | ||
77 | | MergeAudioTranscodeOptions | ||
78 | | OnlyAudioTranscodeOptions | ||
79 | | QuickTranscodeOptions | ||
80 | |||
81 | // --------------------------------------------------------------------------- | ||
82 | |||
83 | const builders: { | ||
84 | [ type in TranscodeVODOptionsType ]: (c: FfmpegCommand, o?: TranscodeVODOptions) => Promise<FfmpegCommand> | FfmpegCommand | ||
85 | } = { | ||
86 | 'quick-transcode': buildQuickTranscodeCommand, | ||
87 | 'hls': buildHLSVODCommand, | ||
88 | 'hls-from-ts': buildHLSVODFromTSCommand, | ||
89 | 'merge-audio': buildAudioMergeCommand, | ||
90 | 'only-audio': buildOnlyAudioCommand, | ||
91 | 'video': buildVODCommand | ||
92 | } | ||
93 | |||
94 | async function transcodeVOD (options: TranscodeVODOptions) { | ||
95 | logger.debug('Will run transcode.', { options, ...lTags() }) | ||
96 | |||
97 | let command = getFFmpeg(options.inputPath, 'vod') | ||
98 | .output(options.outputPath) | ||
99 | |||
100 | command = await builders[options.type](command, options) | ||
101 | |||
102 | command.on('start', () => { | ||
103 | setTimeout(() => { | ||
104 | options.inputFileMutexReleaser() | ||
105 | }, 1000) | ||
106 | }) | ||
107 | |||
108 | await runCommand({ command, job: options.job }) | ||
109 | |||
110 | await fixHLSPlaylistIfNeeded(options) | ||
111 | } | ||
112 | |||
113 | // --------------------------------------------------------------------------- | ||
114 | |||
115 | export { | ||
116 | transcodeVOD, | ||
117 | |||
118 | buildVODCommand, | ||
119 | |||
120 | TranscodeVODOptions, | ||
121 | TranscodeVODOptionsType | ||
122 | } | ||
123 | |||
124 | // --------------------------------------------------------------------------- | ||
125 | |||
126 | async function buildVODCommand (command: FfmpegCommand, options: TranscodeVODOptions) { | ||
127 | const probe = await ffprobePromise(options.inputPath) | ||
128 | |||
129 | let fps = await getVideoStreamFPS(options.inputPath, probe) | ||
130 | fps = computeFPS(fps, options.resolution) | ||
131 | |||
132 | let scaleFilterValue: string | ||
133 | |||
134 | if (options.resolution !== undefined) { | ||
135 | const videoStreamInfo = await getVideoStreamDimensionsInfo(options.inputPath, probe) | ||
136 | |||
137 | scaleFilterValue = videoStreamInfo?.isPortraitMode === true | ||
138 | ? `w=${options.resolution}:h=-2` | ||
139 | : `w=-2:h=${options.resolution}` | ||
140 | } | ||
141 | |||
142 | command = await presetVOD({ | ||
143 | ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]), | ||
144 | |||
145 | command, | ||
146 | input: options.inputPath, | ||
147 | canCopyAudio: true, | ||
148 | canCopyVideo: true, | ||
149 | fps, | ||
150 | scaleFilterValue | ||
151 | }) | ||
152 | |||
153 | return command | ||
154 | } | ||
155 | |||
156 | function buildQuickTranscodeCommand (command: FfmpegCommand) { | ||
157 | command = presetCopy(command) | ||
158 | |||
159 | command = command.outputOption('-map_metadata -1') // strip all metadata | ||
160 | .outputOption('-movflags faststart') | ||
161 | |||
162 | return command | ||
163 | } | ||
164 | |||
165 | // --------------------------------------------------------------------------- | ||
166 | // Audio transcoding | ||
167 | // --------------------------------------------------------------------------- | ||
168 | |||
169 | async function buildAudioMergeCommand (command: FfmpegCommand, options: MergeAudioTranscodeOptions) { | ||
170 | command = command.loop(undefined) | ||
171 | |||
172 | const scaleFilterValue = getMergeAudioScaleFilterValue() | ||
173 | command = await presetVOD({ | ||
174 | ...pick(options, [ 'resolution', 'availableEncoders', 'profile' ]), | ||
175 | |||
176 | command, | ||
177 | input: options.audioPath, | ||
178 | canCopyAudio: true, | ||
179 | canCopyVideo: true, | ||
180 | fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, | ||
181 | scaleFilterValue | ||
182 | }) | ||
183 | |||
184 | command.outputOption('-preset:v veryfast') | ||
185 | |||
186 | command = command.input(options.audioPath) | ||
187 | .outputOption('-tune stillimage') | ||
188 | .outputOption('-shortest') | ||
189 | |||
190 | return command | ||
191 | } | ||
192 | |||
193 | function buildOnlyAudioCommand (command: FfmpegCommand, _options: OnlyAudioTranscodeOptions) { | ||
194 | command = presetOnlyAudio(command) | ||
195 | |||
196 | return command | ||
197 | } | ||
198 | |||
199 | // --------------------------------------------------------------------------- | ||
200 | // HLS transcoding | ||
201 | // --------------------------------------------------------------------------- | ||
202 | |||
203 | async function buildHLSVODCommand (command: FfmpegCommand, options: HLSTranscodeOptions) { | ||
204 | const videoPath = getHLSVideoPath(options) | ||
205 | |||
206 | if (options.copyCodecs) command = presetCopy(command) | ||
207 | else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command) | ||
208 | else command = await buildVODCommand(command, options) | ||
209 | |||
210 | addCommonHLSVODCommandOptions(command, videoPath) | ||
211 | |||
212 | return command | ||
213 | } | ||
214 | |||
215 | function buildHLSVODFromTSCommand (command: FfmpegCommand, options: HLSFromTSTranscodeOptions) { | ||
216 | const videoPath = getHLSVideoPath(options) | ||
217 | |||
218 | command.outputOption('-c copy') | ||
219 | |||
220 | if (options.isAAC) { | ||
221 | // Required for example when copying an AAC stream from an MPEG-TS | ||
222 | // Since it's a bitstream filter, we don't need to reencode the audio | ||
223 | command.outputOption('-bsf:a aac_adtstoasc') | ||
224 | } | ||
225 | |||
226 | addCommonHLSVODCommandOptions(command, videoPath) | ||
227 | |||
228 | return command | ||
229 | } | ||
230 | |||
231 | function addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { | ||
232 | return command.outputOption('-hls_time 4') | ||
233 | .outputOption('-hls_list_size 0') | ||
234 | .outputOption('-hls_playlist_type vod') | ||
235 | .outputOption('-hls_segment_filename ' + outputPath) | ||
236 | .outputOption('-hls_segment_type fmp4') | ||
237 | .outputOption('-f hls') | ||
238 | .outputOption('-hls_flags single_file') | ||
239 | } | ||
240 | |||
241 | async function fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { | ||
242 | if (options.type !== 'hls' && options.type !== 'hls-from-ts') return | ||
243 | |||
244 | const fileContent = await readFile(options.outputPath) | ||
245 | |||
246 | const videoFileName = options.hlsPlaylist.videoFilename | ||
247 | const videoFilePath = getHLSVideoPath(options) | ||
248 | |||
249 | // Fix wrong mapping with some ffmpeg versions | ||
250 | const newContent = fileContent.toString() | ||
251 | .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) | ||
252 | |||
253 | await writeFile(options.outputPath, newContent) | ||
254 | } | ||
255 | |||
256 | // --------------------------------------------------------------------------- | ||
257 | // Helpers | ||
258 | // --------------------------------------------------------------------------- | ||
259 | |||
260 | function getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { | ||
261 | return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` | ||
262 | } | ||
263 | |||
264 | // Avoid "height not divisible by 2" error | ||
265 | function getMergeAudioScaleFilterValue () { | ||
266 | return 'trunc(iw/2)*2:trunc(ih/2)*2' | ||
267 | } | ||
diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts deleted file mode 100644 index fb270b3cb..000000000 --- a/server/helpers/ffmpeg/ffprobe-utils.ts +++ /dev/null | |||
@@ -1,254 +0,0 @@ | |||
1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
2 | import { getMaxBitrate } from '@shared/core-utils' | ||
3 | import { | ||
4 | buildFileMetadata, | ||
5 | ffprobePromise, | ||
6 | getAudioStream, | ||
7 | getMaxAudioBitrate, | ||
8 | getVideoStream, | ||
9 | getVideoStreamBitrate, | ||
10 | getVideoStreamDimensionsInfo, | ||
11 | getVideoStreamDuration, | ||
12 | getVideoStreamFPS, | ||
13 | hasAudioStream | ||
14 | } from '@shared/extra-utils/ffprobe' | ||
15 | import { VideoResolution, VideoTranscodingFPS } from '@shared/models' | ||
16 | import { CONFIG } from '../../initializers/config' | ||
17 | import { VIDEO_TRANSCODING_FPS } from '../../initializers/constants' | ||
18 | import { toEven } from '../core-utils' | ||
19 | import { logger } from '../logger' | ||
20 | |||
21 | /** | ||
22 | * | ||
23 | * Helpers to run ffprobe and extract data from the JSON output | ||
24 | * | ||
25 | */ | ||
26 | |||
27 | // --------------------------------------------------------------------------- | ||
28 | // Codecs | ||
29 | // --------------------------------------------------------------------------- | ||
30 | |||
31 | async function getVideoStreamCodec (path: string) { | ||
32 | const videoStream = await getVideoStream(path) | ||
33 | if (!videoStream) return '' | ||
34 | |||
35 | const videoCodec = videoStream.codec_tag_string | ||
36 | |||
37 | if (videoCodec === 'vp09') return 'vp09.00.50.08' | ||
38 | if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' | ||
39 | |||
40 | const baseProfileMatrix = { | ||
41 | avc1: { | ||
42 | High: '6400', | ||
43 | Main: '4D40', | ||
44 | Baseline: '42E0' | ||
45 | }, | ||
46 | av01: { | ||
47 | High: '1', | ||
48 | Main: '0', | ||
49 | Professional: '2' | ||
50 | } | ||
51 | } | ||
52 | |||
53 | let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] | ||
54 | if (!baseProfile) { | ||
55 | logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) | ||
56 | baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback | ||
57 | } | ||
58 | |||
59 | if (videoCodec === 'av01') { | ||
60 | let level = videoStream.level.toString() | ||
61 | if (level.length === 1) level = `0${level}` | ||
62 | |||
63 | // Guess the tier indicator and bit depth | ||
64 | return `${videoCodec}.${baseProfile}.${level}M.08` | ||
65 | } | ||
66 | |||
67 | let level = videoStream.level.toString(16) | ||
68 | if (level.length === 1) level = `0${level}` | ||
69 | |||
70 | // Default, h264 codec | ||
71 | return `${videoCodec}.${baseProfile}${level}` | ||
72 | } | ||
73 | |||
74 | async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { | ||
75 | const { audioStream } = await getAudioStream(path, existingProbe) | ||
76 | |||
77 | if (!audioStream) return '' | ||
78 | |||
79 | const audioCodecName = audioStream.codec_name | ||
80 | |||
81 | if (audioCodecName === 'opus') return 'opus' | ||
82 | if (audioCodecName === 'vorbis') return 'vorbis' | ||
83 | if (audioCodecName === 'aac') return 'mp4a.40.2' | ||
84 | if (audioCodecName === 'mp3') return 'mp4a.40.34' | ||
85 | |||
86 | logger.warn('Cannot get audio codec of %s.', path, { audioStream }) | ||
87 | |||
88 | return 'mp4a.40.2' // Fallback | ||
89 | } | ||
90 | |||
91 | // --------------------------------------------------------------------------- | ||
92 | // Resolutions | ||
93 | // --------------------------------------------------------------------------- | ||
94 | |||
95 | function computeResolutionsToTranscode (options: { | ||
96 | input: number | ||
97 | type: 'vod' | 'live' | ||
98 | includeInput: boolean | ||
99 | strictLower: boolean | ||
100 | hasAudio: boolean | ||
101 | }) { | ||
102 | const { input, type, includeInput, strictLower, hasAudio } = options | ||
103 | |||
104 | const configResolutions = type === 'vod' | ||
105 | ? CONFIG.TRANSCODING.RESOLUTIONS | ||
106 | : CONFIG.LIVE.TRANSCODING.RESOLUTIONS | ||
107 | |||
108 | const resolutionsEnabled = new Set<number>() | ||
109 | |||
110 | // Put in the order we want to proceed jobs | ||
111 | const availableResolutions: VideoResolution[] = [ | ||
112 | VideoResolution.H_NOVIDEO, | ||
113 | VideoResolution.H_480P, | ||
114 | VideoResolution.H_360P, | ||
115 | VideoResolution.H_720P, | ||
116 | VideoResolution.H_240P, | ||
117 | VideoResolution.H_144P, | ||
118 | VideoResolution.H_1080P, | ||
119 | VideoResolution.H_1440P, | ||
120 | VideoResolution.H_4K | ||
121 | ] | ||
122 | |||
123 | for (const resolution of availableResolutions) { | ||
124 | // Resolution not enabled | ||
125 | if (configResolutions[resolution + 'p'] !== true) continue | ||
126 | // Too big resolution for input file | ||
127 | if (input < resolution) continue | ||
128 | // We only want lower resolutions than input file | ||
129 | if (strictLower && input === resolution) continue | ||
130 | // Audio resolutio but no audio in the video | ||
131 | if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue | ||
132 | |||
133 | resolutionsEnabled.add(resolution) | ||
134 | } | ||
135 | |||
136 | if (includeInput) { | ||
137 | // Always use an even resolution to avoid issues with ffmpeg | ||
138 | resolutionsEnabled.add(toEven(input)) | ||
139 | } | ||
140 | |||
141 | return Array.from(resolutionsEnabled) | ||
142 | } | ||
143 | |||
144 | // --------------------------------------------------------------------------- | ||
145 | // Can quick transcode | ||
146 | // --------------------------------------------------------------------------- | ||
147 | |||
148 | async function canDoQuickTranscode (path: string): Promise<boolean> { | ||
149 | if (CONFIG.TRANSCODING.PROFILE !== 'default') return false | ||
150 | |||
151 | const probe = await ffprobePromise(path) | ||
152 | |||
153 | return await canDoQuickVideoTranscode(path, probe) && | ||
154 | await canDoQuickAudioTranscode(path, probe) | ||
155 | } | ||
156 | |||
157 | async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> { | ||
158 | const parsedAudio = await getAudioStream(path, probe) | ||
159 | |||
160 | if (!parsedAudio.audioStream) return true | ||
161 | |||
162 | if (parsedAudio.audioStream['codec_name'] !== 'aac') return false | ||
163 | |||
164 | const audioBitrate = parsedAudio.bitrate | ||
165 | if (!audioBitrate) return false | ||
166 | |||
167 | const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) | ||
168 | if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false | ||
169 | |||
170 | const channelLayout = parsedAudio.audioStream['channel_layout'] | ||
171 | // Causes playback issues with Chrome | ||
172 | if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false | ||
173 | |||
174 | return true | ||
175 | } | ||
176 | |||
177 | async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise<boolean> { | ||
178 | const videoStream = await getVideoStream(path, probe) | ||
179 | const fps = await getVideoStreamFPS(path, probe) | ||
180 | const bitRate = await getVideoStreamBitrate(path, probe) | ||
181 | const resolutionData = await getVideoStreamDimensionsInfo(path, probe) | ||
182 | |||
183 | // If ffprobe did not manage to guess the bitrate | ||
184 | if (!bitRate) return false | ||
185 | |||
186 | // check video params | ||
187 | if (!videoStream) return false | ||
188 | if (videoStream['codec_name'] !== 'h264') return false | ||
189 | if (videoStream['pix_fmt'] !== 'yuv420p') return false | ||
190 | if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false | ||
191 | if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false | ||
192 | |||
193 | return true | ||
194 | } | ||
195 | |||
196 | // --------------------------------------------------------------------------- | ||
197 | // Framerate | ||
198 | // --------------------------------------------------------------------------- | ||
199 | |||
200 | function getClosestFramerateStandard <K extends keyof Pick<VideoTranscodingFPS, 'HD_STANDARD' | 'STANDARD'>> (fps: number, type: K) { | ||
201 | return VIDEO_TRANSCODING_FPS[type].slice(0) | ||
202 | .sort((a, b) => fps % a - fps % b)[0] | ||
203 | } | ||
204 | |||
205 | function computeFPS (fpsArg: number, resolution: VideoResolution) { | ||
206 | let fps = fpsArg | ||
207 | |||
208 | if ( | ||
209 | // On small/medium resolutions, limit FPS | ||
210 | resolution !== undefined && | ||
211 | resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && | ||
212 | fps > VIDEO_TRANSCODING_FPS.AVERAGE | ||
213 | ) { | ||
214 | // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value | ||
215 | fps = getClosestFramerateStandard(fps, 'STANDARD') | ||
216 | } | ||
217 | |||
218 | // Hard FPS limits | ||
219 | if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD') | ||
220 | |||
221 | if (fps < VIDEO_TRANSCODING_FPS.MIN) { | ||
222 | throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) | ||
223 | } | ||
224 | |||
225 | return fps | ||
226 | } | ||
227 | |||
228 | // --------------------------------------------------------------------------- | ||
229 | |||
230 | export { | ||
231 | // Re export ffprobe utils | ||
232 | getVideoStreamDimensionsInfo, | ||
233 | buildFileMetadata, | ||
234 | getMaxAudioBitrate, | ||
235 | getVideoStream, | ||
236 | getVideoStreamDuration, | ||
237 | getAudioStream, | ||
238 | hasAudioStream, | ||
239 | getVideoStreamFPS, | ||
240 | ffprobePromise, | ||
241 | getVideoStreamBitrate, | ||
242 | |||
243 | getVideoStreamCodec, | ||
244 | getAudioStreamCodec, | ||
245 | |||
246 | computeFPS, | ||
247 | getClosestFramerateStandard, | ||
248 | |||
249 | computeResolutionsToTranscode, | ||
250 | |||
251 | canDoQuickTranscode, | ||
252 | canDoQuickVideoTranscode, | ||
253 | canDoQuickAudioTranscode | ||
254 | } | ||
diff --git a/server/helpers/ffmpeg/framerate.ts b/server/helpers/ffmpeg/framerate.ts new file mode 100644 index 000000000..18cb0e0e2 --- /dev/null +++ b/server/helpers/ffmpeg/framerate.ts | |||
@@ -0,0 +1,44 @@ | |||
1 | import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
2 | import { VideoResolution } from '@shared/models' | ||
3 | |||
4 | export function computeOutputFPS (options: { | ||
5 | inputFPS: number | ||
6 | resolution: VideoResolution | ||
7 | }) { | ||
8 | const { resolution } = options | ||
9 | |||
10 | let fps = options.inputFPS | ||
11 | |||
12 | if ( | ||
13 | // On small/medium resolutions, limit FPS | ||
14 | resolution !== undefined && | ||
15 | resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && | ||
16 | fps > VIDEO_TRANSCODING_FPS.AVERAGE | ||
17 | ) { | ||
18 | // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value | ||
19 | fps = getClosestFramerateStandard({ fps, type: 'STANDARD' }) | ||
20 | } | ||
21 | |||
22 | // Hard FPS limits | ||
23 | if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' }) | ||
24 | |||
25 | if (fps < VIDEO_TRANSCODING_FPS.MIN) { | ||
26 | throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) | ||
27 | } | ||
28 | |||
29 | return fps | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | // Private | ||
34 | // --------------------------------------------------------------------------- | ||
35 | |||
36 | function getClosestFramerateStandard (options: { | ||
37 | fps: number | ||
38 | type: 'HD_STANDARD' | 'STANDARD' | ||
39 | }) { | ||
40 | const { fps, type } = options | ||
41 | |||
42 | return VIDEO_TRANSCODING_FPS[type].slice(0) | ||
43 | .sort((a, b) => fps % a - fps % b)[0] | ||
44 | } | ||
diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts index e3bb2013f..bf1c73fb6 100644 --- a/server/helpers/ffmpeg/index.ts +++ b/server/helpers/ffmpeg/index.ts | |||
@@ -1,8 +1,4 @@ | |||
1 | export * from './ffmpeg-commons' | 1 | export * from './codecs' |
2 | export * from './ffmpeg-edition' | 2 | export * from './ffmpeg-image' |
3 | export * from './ffmpeg-encoders' | 3 | export * from './ffmpeg-options' |
4 | export * from './ffmpeg-images' | 4 | export * from './framerate' |
5 | export * from './ffmpeg-live' | ||
6 | export * from './ffmpeg-presets' | ||
7 | export * from './ffmpeg-vod' | ||
8 | export * from './ffprobe-utils' | ||
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index bbd4692ef..05b258d8a 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts | |||
@@ -3,7 +3,7 @@ import Jimp, { read as jimpRead } from 'jimp' | |||
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { getLowercaseExtension } from '@shared/core-utils' | 4 | import { getLowercaseExtension } from '@shared/core-utils' |
5 | import { buildUUID } from '@shared/extra-utils' | 5 | import { buildUUID } from '@shared/extra-utils' |
6 | import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg/ffmpeg-images' | 6 | import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg' |
7 | import { logger, loggerTagsFactory } from './logger' | 7 | import { logger, loggerTagsFactory } from './logger' |
8 | 8 | ||
9 | const lTags = loggerTagsFactory('image-utils') | 9 | const lTags = loggerTagsFactory('image-utils') |
@@ -30,7 +30,7 @@ async function processImage (options: { | |||
30 | 30 | ||
31 | // Use FFmpeg to process GIF | 31 | // Use FFmpeg to process GIF |
32 | if (extension === '.gif') { | 32 | if (extension === '.gif') { |
33 | await processGIF(path, destination, newSize) | 33 | await processGIF({ path, destination, newSize }) |
34 | } else { | 34 | } else { |
35 | await jimpProcessor(path, destination, newSize, extension) | 35 | await jimpProcessor(path, destination, newSize, extension) |
36 | } | 36 | } |
@@ -50,7 +50,7 @@ async function generateImageFromVideoFile (options: { | |||
50 | const pendingImagePath = join(folder, pendingImageName) | 50 | const pendingImagePath = join(folder, pendingImageName) |
51 | 51 | ||
52 | try { | 52 | try { |
53 | await generateThumbnailFromVideo(fromPath, folder, imageName) | 53 | await generateThumbnailFromVideo({ fromPath, folder, imageName }) |
54 | 54 | ||
55 | const destination = join(folder, imageName) | 55 | const destination = join(folder, imageName) |
56 | await processImage({ path: pendingImagePath, destination, newSize: size }) | 56 | await processImage({ path: pendingImagePath, destination, newSize: size }) |
@@ -99,7 +99,7 @@ async function jimpProcessor (path: string, destination: string, newSize: { widt | |||
99 | logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err }) | 99 | logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err }) |
100 | 100 | ||
101 | const newName = path + '.jpg' | 101 | const newName = path + '.jpg' |
102 | await convertWebPToJPG(path, newName) | 102 | await convertWebPToJPG({ path, destination: newName }) |
103 | await rename(newName, path) | 103 | await rename(newName, path) |
104 | 104 | ||
105 | sourceImage = await jimpRead(path) | 105 | sourceImage = await jimpRead(path) |
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index ae7d11800..95e78a904 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts | |||
@@ -2,10 +2,11 @@ import { compare, genSalt, hash } from 'bcrypt' | |||
2 | import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' | 2 | import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' |
3 | import { Request } from 'express' | 3 | import { Request } from 'express' |
4 | import { cloneDeep } from 'lodash' | 4 | import { cloneDeep } from 'lodash' |
5 | import { promisify1, promisify2 } from '@shared/core-utils' | ||
5 | import { sha256 } from '@shared/extra-utils' | 6 | import { sha256 } from '@shared/extra-utils' |
6 | import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' | 7 | import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' |
7 | import { MActor } from '../types/models' | 8 | import { MActor } from '../types/models' |
8 | import { generateRSAKeyPairPromise, promisify1, promisify2, randomBytesPromise, scryptPromise } from './core-utils' | 9 | import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils' |
9 | import { jsonld } from './custom-jsonld-signature' | 10 | import { jsonld } from './custom-jsonld-signature' |
10 | import { logger } from './logger' | 11 | import { logger } from './logger' |
11 | 12 | ||
diff --git a/server/helpers/token-generator.ts b/server/helpers/token-generator.ts new file mode 100644 index 000000000..16313b818 --- /dev/null +++ b/server/helpers/token-generator.ts | |||
@@ -0,0 +1,19 @@ | |||
1 | import { buildUUID } from '@shared/extra-utils' | ||
2 | |||
3 | function generateRunnerRegistrationToken () { | ||
4 | return 'ptrrt-' + buildUUID() | ||
5 | } | ||
6 | |||
7 | function generateRunnerToken () { | ||
8 | return 'ptrt-' + buildUUID() | ||
9 | } | ||
10 | |||
11 | function generateRunnerJobToken () { | ||
12 | return 'ptrjt-' + buildUUID() | ||
13 | } | ||
14 | |||
15 | export { | ||
16 | generateRunnerRegistrationToken, | ||
17 | generateRunnerToken, | ||
18 | generateRunnerJobToken | ||
19 | } | ||
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index a3c93e6fe..e690e3890 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -13,9 +13,9 @@ import { VideoPathManager } from '@server/lib/video-path-manager' | |||
13 | import { MVideo } from '@server/types/models/video/video' | 13 | import { MVideo } from '@server/types/models/video/video' |
14 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' | 14 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' |
15 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' | 15 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' |
16 | import { promisify2 } from '@shared/core-utils' | ||
16 | import { sha1 } from '@shared/extra-utils' | 17 | import { sha1 } from '@shared/extra-utils' |
17 | import { CONFIG } from '../initializers/config' | 18 | import { CONFIG } from '../initializers/config' |
18 | import { promisify2 } from './core-utils' | ||
19 | import { logger } from './logger' | 19 | import { logger } from './logger' |
20 | import { generateVideoImportTmpPath } from './utils' | 20 | import { generateVideoImportTmpPath } from './utils' |
21 | import { extractVideo } from './video' | 21 | import { extractVideo } from './video' |
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index 14ed82cb4..68dea909d 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import config from 'config' | 1 | import config from 'config' |
2 | import { URL } from 'url' | 2 | import { URL } from 'url' |
3 | import { getFFmpegVersion } from '@server/helpers/ffmpeg' | ||
4 | import { uniqify } from '@shared/core-utils' | 3 | import { uniqify } from '@shared/core-utils' |
4 | import { getFFmpegVersion } from '@shared/ffmpeg' | ||
5 | import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' | 5 | import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' |
6 | import { RecentlyAddedStrategy } from '../../shared/models/redundancy' | 6 | import { RecentlyAddedStrategy } from '../../shared/models/redundancy' |
7 | import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils' | 7 | import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils' |
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 49010c059..2361aa1eb 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import { IConfig } from 'config' | 1 | import { IConfig } from 'config' |
2 | import { parseSemVersion, promisify0 } from '../helpers/core-utils' | 2 | import { promisify0 } from '@shared/core-utils' |
3 | import { parseSemVersion } from '../helpers/core-utils' | ||
3 | import { logger } from '../helpers/logger' | 4 | import { logger } from '../helpers/logger' |
4 | 5 | ||
5 | // Special behaviour for config because we can reload it | 6 | // Special behaviour for config because we can reload it |
@@ -36,7 +37,9 @@ function checkMissedConfig () { | |||
36 | 'transcoding.profile', 'transcoding.concurrency', | 37 | 'transcoding.profile', 'transcoding.concurrency', |
37 | 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', | 38 | 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', |
38 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', | 39 | 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', |
39 | 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled', | 40 | 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', |
41 | 'video_studio.enabled', | ||
42 | 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', | ||
40 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', | 43 | 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', |
41 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', | 44 | 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', |
42 | 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', | 45 | 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', |
@@ -74,7 +77,8 @@ function checkMissedConfig () { | |||
74 | 'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile', | 77 | 'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile', |
75 | 'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', | 78 | 'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', |
76 | 'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', | 79 | 'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', |
77 | 'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution' | 80 | 'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution', |
81 | 'live.transcoding.remote_runners.enabled' | ||
78 | ] | 82 | ] |
79 | 83 | ||
80 | const requiredAlternatives = [ | 84 | const requiredAlternatives = [ |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index e2442213c..699dd4704 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -304,6 +304,12 @@ const CONFIG = { | |||
304 | COUNT: config.get<number>('feeds.comments.count') | 304 | COUNT: config.get<number>('feeds.comments.count') |
305 | } | 305 | } |
306 | }, | 306 | }, |
307 | REMOTE_RUNNERS: { | ||
308 | STALLED_JOBS: { | ||
309 | LIVE: parseDurationToMs(config.get<string>('remote_runners.stalled_jobs.live')), | ||
310 | VOD: parseDurationToMs(config.get<string>('remote_runners.stalled_jobs.vod')) | ||
311 | } | ||
312 | }, | ||
307 | ADMIN: { | 313 | ADMIN: { |
308 | get EMAIL () { return config.get<string>('admin.email') } | 314 | get EMAIL () { return config.get<string>('admin.email') } |
309 | }, | 315 | }, |
@@ -359,6 +365,9 @@ const CONFIG = { | |||
359 | }, | 365 | }, |
360 | WEBTORRENT: { | 366 | WEBTORRENT: { |
361 | get ENABLED () { return config.get<boolean>('transcoding.webtorrent.enabled') } | 367 | get ENABLED () { return config.get<boolean>('transcoding.webtorrent.enabled') } |
368 | }, | ||
369 | REMOTE_RUNNERS: { | ||
370 | get ENABLED () { return config.get<boolean>('transcoding.remote_runners.enabled') } | ||
362 | } | 371 | } |
363 | }, | 372 | }, |
364 | LIVE: { | 373 | LIVE: { |
@@ -406,6 +415,9 @@ const CONFIG = { | |||
406 | get '1080p' () { return config.get<boolean>('live.transcoding.resolutions.1080p') }, | 415 | get '1080p' () { return config.get<boolean>('live.transcoding.resolutions.1080p') }, |
407 | get '1440p' () { return config.get<boolean>('live.transcoding.resolutions.1440p') }, | 416 | get '1440p' () { return config.get<boolean>('live.transcoding.resolutions.1440p') }, |
408 | get '2160p' () { return config.get<boolean>('live.transcoding.resolutions.2160p') } | 417 | get '2160p' () { return config.get<boolean>('live.transcoding.resolutions.2160p') } |
418 | }, | ||
419 | REMOTE_RUNNERS: { | ||
420 | get ENABLED () { return config.get<boolean>('live.transcoding.remote_runners.enabled') } | ||
409 | } | 421 | } |
410 | } | 422 | } |
411 | }, | 423 | }, |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6cad4eb23..279e77421 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils' | |||
6 | import { | 6 | import { |
7 | AbuseState, | 7 | AbuseState, |
8 | JobType, | 8 | JobType, |
9 | RunnerJobState, | ||
9 | UserRegistrationState, | 10 | UserRegistrationState, |
10 | VideoChannelSyncState, | 11 | VideoChannelSyncState, |
11 | VideoImportState, | 12 | VideoImportState, |
@@ -26,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
26 | 27 | ||
27 | // --------------------------------------------------------------------------- | 28 | // --------------------------------------------------------------------------- |
28 | 29 | ||
29 | const LAST_MIGRATION_VERSION = 760 | 30 | const LAST_MIGRATION_VERSION = 765 |
30 | 31 | ||
31 | // --------------------------------------------------------------------------- | 32 | // --------------------------------------------------------------------------- |
32 | 33 | ||
@@ -81,6 +82,10 @@ const SORTABLE_COLUMNS = { | |||
81 | 82 | ||
82 | USER_REGISTRATIONS: [ 'createdAt', 'state' ], | 83 | USER_REGISTRATIONS: [ 'createdAt', 'state' ], |
83 | 84 | ||
85 | RUNNERS: [ 'createdAt' ], | ||
86 | RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ], | ||
87 | RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ], | ||
88 | |||
84 | VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ], | 89 | VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ], |
85 | 90 | ||
86 | // Don't forget to update peertube-search-index with the same values | 91 | // Don't forget to update peertube-search-index with the same values |
@@ -139,6 +144,8 @@ const REMOTE_SCHEME = { | |||
139 | WS: 'wss' | 144 | WS: 'wss' |
140 | } | 145 | } |
141 | 146 | ||
147 | // --------------------------------------------------------------------------- | ||
148 | |||
142 | const JOB_ATTEMPTS: { [id in JobType]: number } = { | 149 | const JOB_ATTEMPTS: { [id in JobType]: number } = { |
143 | 'activitypub-http-broadcast': 1, | 150 | 'activitypub-http-broadcast': 1, |
144 | 'activitypub-http-broadcast-parallel': 1, | 151 | 'activitypub-http-broadcast-parallel': 1, |
@@ -160,6 +167,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
160 | 'video-channel-import': 1, | 167 | 'video-channel-import': 1, |
161 | 'after-video-channel-import': 1, | 168 | 'after-video-channel-import': 1, |
162 | 'move-to-object-storage': 3, | 169 | 'move-to-object-storage': 3, |
170 | 'transcoding-job-builder': 1, | ||
163 | 'notify': 1, | 171 | 'notify': 1, |
164 | 'federate-video': 1 | 172 | 'federate-video': 1 |
165 | } | 173 | } |
@@ -183,6 +191,7 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im | |||
183 | 'move-to-object-storage': 1, | 191 | 'move-to-object-storage': 1, |
184 | 'video-channel-import': 1, | 192 | 'video-channel-import': 1, |
185 | 'after-video-channel-import': 1, | 193 | 'after-video-channel-import': 1, |
194 | 'transcoding-job-builder': 1, | ||
186 | 'notify': 5, | 195 | 'notify': 5, |
187 | 'federate-video': 3 | 196 | 'federate-video': 3 |
188 | } | 197 | } |
@@ -207,6 +216,7 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
207 | 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours | 216 | 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours |
208 | 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours | 217 | 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours |
209 | 'after-video-channel-import': 60000 * 5, // 5 minutes | 218 | 'after-video-channel-import': 60000 * 5, // 5 minutes |
219 | 'transcoding-job-builder': 60000, // 1 minute | ||
210 | 'notify': 60000 * 5, // 5 minutes | 220 | 'notify': 60000 * 5, // 5 minutes |
211 | 'federate-video': 60000 * 5 // 5 minutes | 221 | 'federate-video': 60000 * 5 // 5 minutes |
212 | } | 222 | } |
@@ -222,21 +232,6 @@ const JOB_PRIORITY = { | |||
222 | TRANSCODING: 100 | 232 | TRANSCODING: 100 |
223 | } | 233 | } |
224 | 234 | ||
225 | const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job | ||
226 | const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) | ||
227 | |||
228 | const AP_CLEANER = { | ||
229 | CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job | ||
230 | UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource | ||
231 | PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS | ||
232 | } | ||
233 | |||
234 | const REQUEST_TIMEOUTS = { | ||
235 | DEFAULT: 7000, // 7 seconds | ||
236 | FILE: 30000, // 30 seconds | ||
237 | REDUNDANCY: JOB_TTL['video-redundancy'] | ||
238 | } | ||
239 | |||
240 | const JOB_REMOVAL_OPTIONS = { | 235 | const JOB_REMOVAL_OPTIONS = { |
241 | COUNT: 10000, // Max jobs to store | 236 | COUNT: 10000, // Max jobs to store |
242 | 237 | ||
@@ -256,7 +251,29 @@ const JOB_REMOVAL_OPTIONS = { | |||
256 | 251 | ||
257 | const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) | 252 | const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) |
258 | 253 | ||
254 | const RUNNER_JOBS = { | ||
255 | MAX_FAILURES: 5 | ||
256 | } | ||
257 | |||
258 | // --------------------------------------------------------------------------- | ||
259 | |||
260 | const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job | ||
261 | const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) | ||
262 | |||
263 | const AP_CLEANER = { | ||
264 | CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job | ||
265 | UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource | ||
266 | PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS | ||
267 | } | ||
268 | |||
269 | const REQUEST_TIMEOUTS = { | ||
270 | DEFAULT: 7000, // 7 seconds | ||
271 | FILE: 30000, // 30 seconds | ||
272 | REDUNDANCY: JOB_TTL['video-redundancy'] | ||
273 | } | ||
274 | |||
259 | const SCHEDULER_INTERVALS_MS = { | 275 | const SCHEDULER_INTERVALS_MS = { |
276 | RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE), | ||
260 | ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour | 277 | ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour |
261 | REMOVE_OLD_JOBS: 60000 * 60, // 1 hour | 278 | REMOVE_OLD_JOBS: 60000 * 60, // 1 hour |
262 | UPDATE_VIDEOS: 60000, // 1 minute | 279 | UPDATE_VIDEOS: 60000, // 1 minute |
@@ -410,6 +427,17 @@ const CONSTRAINTS_FIELDS = { | |||
410 | CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length | 427 | CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length |
411 | CLIENT_META: { min: 1, max: 5000 }, // Length | 428 | CLIENT_META: { min: 1, max: 5000 }, // Length |
412 | CLIENT_USER_AGENT: { min: 1, max: 200 } // Length | 429 | CLIENT_USER_AGENT: { min: 1, max: 200 } // Length |
430 | }, | ||
431 | RUNNERS: { | ||
432 | TOKEN: { min: 1, max: 1000 }, // Length | ||
433 | NAME: { min: 1, max: 100 }, // Length | ||
434 | DESCRIPTION: { min: 1, max: 1000 } // Length | ||
435 | }, | ||
436 | RUNNER_JOBS: { | ||
437 | TOKEN: { min: 1, max: 1000 }, // Length | ||
438 | REASON: { min: 1, max: 5000 }, // Length | ||
439 | ERROR_MESSAGE: { min: 1, max: 5000 }, // Length | ||
440 | PROGRESS: { min: 0, max: 100 } // Value | ||
413 | } | 441 | } |
414 | } | 442 | } |
415 | 443 | ||
@@ -540,6 +568,17 @@ const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = { | |||
540 | [VideoPlaylistType.WATCH_LATER]: 'Watch later' | 568 | [VideoPlaylistType.WATCH_LATER]: 'Watch later' |
541 | } | 569 | } |
542 | 570 | ||
571 | const RUNNER_JOB_STATES: { [ id in RunnerJobState ]: string } = { | ||
572 | [RunnerJobState.PROCESSING]: 'Processing', | ||
573 | [RunnerJobState.COMPLETED]: 'Completed', | ||
574 | [RunnerJobState.PENDING]: 'Pending', | ||
575 | [RunnerJobState.ERRORED]: 'Errored', | ||
576 | [RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish', | ||
577 | [RunnerJobState.CANCELLED]: 'Cancelled', | ||
578 | [RunnerJobState.PARENT_ERRORED]: 'Parent job failed', | ||
579 | [RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled' | ||
580 | } | ||
581 | |||
543 | const MIMETYPES = { | 582 | const MIMETYPES = { |
544 | AUDIO: { | 583 | AUDIO: { |
545 | MIMETYPE_EXT: { | 584 | MIMETYPE_EXT: { |
@@ -594,6 +633,11 @@ const MIMETYPES = { | |||
594 | MIMETYPE_EXT: { | 633 | MIMETYPE_EXT: { |
595 | 'application/x-bittorrent': '.torrent' | 634 | 'application/x-bittorrent': '.torrent' |
596 | } | 635 | } |
636 | }, | ||
637 | M3U8: { | ||
638 | MIMETYPE_EXT: { | ||
639 | 'application/vnd.apple.mpegurl': '.m3u8' | ||
640 | } | ||
597 | } | 641 | } |
598 | } | 642 | } |
599 | MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT) | 643 | MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT) |
@@ -1027,6 +1071,7 @@ export { | |||
1027 | SEARCH_INDEX, | 1071 | SEARCH_INDEX, |
1028 | DIRECTORIES, | 1072 | DIRECTORIES, |
1029 | RESUMABLE_UPLOAD_SESSION_LIFETIME, | 1073 | RESUMABLE_UPLOAD_SESSION_LIFETIME, |
1074 | RUNNER_JOB_STATES, | ||
1030 | P2P_MEDIA_LOADER_PEER_VERSION, | 1075 | P2P_MEDIA_LOADER_PEER_VERSION, |
1031 | ACTOR_IMAGES_SIZE, | 1076 | ACTOR_IMAGES_SIZE, |
1032 | ACCEPT_HEADERS, | 1077 | ACCEPT_HEADERS, |
@@ -1085,6 +1130,7 @@ export { | |||
1085 | USER_REGISTRATION_STATES, | 1130 | USER_REGISTRATION_STATES, |
1086 | LRU_CACHE, | 1131 | LRU_CACHE, |
1087 | REQUEST_TIMEOUTS, | 1132 | REQUEST_TIMEOUTS, |
1133 | RUNNER_JOBS, | ||
1088 | MAX_LOCAL_VIEWER_WATCH_SECTIONS, | 1134 | MAX_LOCAL_VIEWER_WATCH_SECTIONS, |
1089 | USER_PASSWORD_RESET_LIFETIME, | 1135 | USER_PASSWORD_RESET_LIFETIME, |
1090 | USER_PASSWORD_CREATE_LIFETIME, | 1136 | USER_PASSWORD_CREATE_LIFETIME, |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 3f31099ed..14dd8c379 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -1,6 +1,9 @@ | |||
1 | import { QueryTypes, Transaction } from 'sequelize' | 1 | import { QueryTypes, Transaction } from 'sequelize' |
2 | import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' | 2 | import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' |
3 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | 3 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' |
4 | import { RunnerModel } from '@server/models/runner/runner' | ||
5 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
6 | import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' | ||
4 | import { TrackerModel } from '@server/models/server/tracker' | 7 | import { TrackerModel } from '@server/models/server/tracker' |
5 | import { VideoTrackerModel } from '@server/models/server/video-tracker' | 8 | import { VideoTrackerModel } from '@server/models/server/video-tracker' |
6 | import { UserModel } from '@server/models/user/user' | 9 | import { UserModel } from '@server/models/user/user' |
@@ -9,6 +12,7 @@ import { UserRegistrationModel } from '@server/models/user/user-registration' | |||
9 | import { UserVideoHistoryModel } from '@server/models/user/user-video-history' | 12 | import { UserVideoHistoryModel } from '@server/models/user/user-video-history' |
10 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' | 13 | import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' |
11 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 14 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
15 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
12 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | 16 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' |
13 | import { VideoSourceModel } from '@server/models/video/video-source' | 17 | import { VideoSourceModel } from '@server/models/video/video-source' |
14 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | 18 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' |
@@ -52,7 +56,6 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla | |||
52 | import { VideoTagModel } from '../models/video/video-tag' | 56 | import { VideoTagModel } from '../models/video/video-tag' |
53 | import { VideoViewModel } from '../models/view/video-view' | 57 | import { VideoViewModel } from '../models/view/video-view' |
54 | import { CONFIG } from './config' | 58 | import { CONFIG } from './config' |
55 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
56 | 59 | ||
57 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 60 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
58 | 61 | ||
@@ -159,7 +162,10 @@ async function initDatabaseModels (silent: boolean) { | |||
159 | ActorCustomPageModel, | 162 | ActorCustomPageModel, |
160 | VideoJobInfoModel, | 163 | VideoJobInfoModel, |
161 | VideoChannelSyncModel, | 164 | VideoChannelSyncModel, |
162 | UserRegistrationModel | 165 | UserRegistrationModel, |
166 | RunnerRegistrationTokenModel, | ||
167 | RunnerModel, | ||
168 | RunnerJobModel | ||
163 | ]) | 169 | ]) |
164 | 170 | ||
165 | // Check extensions exist in the database | 171 | // Check extensions exist in the database |
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index f48f348a7..2406a5936 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts | |||
@@ -2,7 +2,9 @@ import { ensureDir, readdir, remove } from 'fs-extra' | |||
2 | import passwordGenerator from 'password-generator' | 2 | import passwordGenerator from 'password-generator' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { isTestOrDevInstance } from '@server/helpers/core-utils' | 4 | import { isTestOrDevInstance } from '@server/helpers/core-utils' |
5 | import { generateRunnerRegistrationToken } from '@server/helpers/token-generator' | ||
5 | import { getNodeABIVersion } from '@server/helpers/version' | 6 | import { getNodeABIVersion } from '@server/helpers/version' |
7 | import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' | ||
6 | import { UserRole } from '@shared/models' | 8 | import { UserRole } from '@shared/models' |
7 | import { logger } from '../helpers/logger' | 9 | import { logger } from '../helpers/logger' |
8 | import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user' | 10 | import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user' |
@@ -22,7 +24,8 @@ async function installApplication () { | |||
22 | return Promise.all([ | 24 | return Promise.all([ |
23 | createApplicationIfNotExist(), | 25 | createApplicationIfNotExist(), |
24 | createOAuthClientIfNotExist(), | 26 | createOAuthClientIfNotExist(), |
25 | createOAuthAdminIfNotExist() | 27 | createOAuthAdminIfNotExist(), |
28 | createRunnerRegistrationTokenIfNotExist() | ||
26 | ]) | 29 | ]) |
27 | }), | 30 | }), |
28 | 31 | ||
@@ -183,3 +186,14 @@ async function createApplicationIfNotExist () { | |||
183 | 186 | ||
184 | return createApplicationActor(application.id) | 187 | return createApplicationActor(application.id) |
185 | } | 188 | } |
189 | |||
190 | async function createRunnerRegistrationTokenIfNotExist () { | ||
191 | const total = await RunnerRegistrationTokenModel.countTotal() | ||
192 | if (total !== 0) return undefined | ||
193 | |||
194 | const token = new RunnerRegistrationTokenModel({ | ||
195 | registrationToken: generateRunnerRegistrationToken() | ||
196 | }) | ||
197 | |||
198 | await token.save() | ||
199 | } | ||
diff --git a/server/initializers/migrations/0765-remote-transcoding.ts b/server/initializers/migrations/0765-remote-transcoding.ts new file mode 100644 index 000000000..40cca03b4 --- /dev/null +++ b/server/initializers/migrations/0765-remote-transcoding.ts | |||
@@ -0,0 +1,78 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction | ||
5 | queryInterface: Sequelize.QueryInterface | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | }): Promise<void> { | ||
8 | { | ||
9 | const query = ` | ||
10 | CREATE TABLE IF NOT EXISTS "runnerRegistrationToken"( | ||
11 | "id" serial, | ||
12 | "registrationToken" varchar(255) NOT NULL, | ||
13 | "createdAt" timestamp with time zone NOT NULL, | ||
14 | "updatedAt" timestamp with time zone NOT NULL, | ||
15 | PRIMARY KEY ("id") | ||
16 | ); | ||
17 | ` | ||
18 | |||
19 | await utils.sequelize.query(query, { transaction : utils.transaction }) | ||
20 | } | ||
21 | |||
22 | { | ||
23 | const query = ` | ||
24 | CREATE TABLE IF NOT EXISTS "runner"( | ||
25 | "id" serial, | ||
26 | "runnerToken" varchar(255) NOT NULL, | ||
27 | "name" varchar(255) NOT NULL, | ||
28 | "description" varchar(1000), | ||
29 | "lastContact" timestamp with time zone NOT NULL, | ||
30 | "ip" varchar(255) NOT NULL, | ||
31 | "runnerRegistrationTokenId" integer REFERENCES "runnerRegistrationToken"("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
32 | "createdAt" timestamp with time zone NOT NULL, | ||
33 | "updatedAt" timestamp with time zone NOT NULL, | ||
34 | PRIMARY KEY ("id") | ||
35 | ); | ||
36 | ` | ||
37 | |||
38 | await utils.sequelize.query(query, { transaction : utils.transaction }) | ||
39 | } | ||
40 | |||
41 | { | ||
42 | const query = ` | ||
43 | CREATE TABLE IF NOT EXISTS "runnerJob"( | ||
44 | "id" serial, | ||
45 | "uuid" uuid NOT NULL, | ||
46 | "type" varchar(255) NOT NULL, | ||
47 | "payload" jsonb NOT NULL, | ||
48 | "privatePayload" jsonb NOT NULL, | ||
49 | "state" integer NOT NULL, | ||
50 | "failures" integer NOT NULL DEFAULT 0, | ||
51 | "error" varchar(5000), | ||
52 | "priority" integer NOT NULL, | ||
53 | "processingJobToken" varchar(255), | ||
54 | "progress" integer, | ||
55 | "startedAt" timestamp with time zone, | ||
56 | "finishedAt" timestamp with time zone, | ||
57 | "dependsOnRunnerJobId" integer REFERENCES "runnerJob"("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
58 | "runnerId" integer REFERENCES "runner"("id") ON DELETE SET NULL ON UPDATE CASCADE, | ||
59 | "createdAt" timestamp with time zone NOT NULL, | ||
60 | "updatedAt" timestamp with time zone NOT NULL, | ||
61 | PRIMARY KEY ("id") | ||
62 | ); | ||
63 | |||
64 | |||
65 | ` | ||
66 | |||
67 | await utils.sequelize.query(query, { transaction : utils.transaction }) | ||
68 | } | ||
69 | } | ||
70 | |||
71 | function down (options) { | ||
72 | throw new Error('Not implemented.') | ||
73 | } | ||
74 | |||
75 | export { | ||
76 | up, | ||
77 | down | ||
78 | } | ||
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 053b5d326..fc1d7e1b0 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -3,10 +3,11 @@ import { flatten } from 'lodash' | |||
3 | import PQueue from 'p-queue' | 3 | import PQueue from 'p-queue' |
4 | import { basename, dirname, join } from 'path' | 4 | import { basename, dirname, join } from 'path' |
5 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' | 5 | import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' |
6 | import { uniqify } from '@shared/core-utils' | 6 | import { uniqify, uuidRegex } from '@shared/core-utils' |
7 | import { sha256 } from '@shared/extra-utils' | 7 | import { sha256 } from '@shared/extra-utils' |
8 | import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' | ||
8 | import { VideoStorage } from '@shared/models' | 9 | import { VideoStorage } from '@shared/models' |
9 | import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' | 10 | import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' |
10 | import { logger } from '../helpers/logger' | 11 | import { logger } from '../helpers/logger' |
11 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | 12 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
12 | import { generateRandomString } from '../helpers/utils' | 13 | import { generateRandomString } from '../helpers/utils' |
@@ -234,6 +235,16 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, | |||
234 | 235 | ||
235 | // --------------------------------------------------------------------------- | 236 | // --------------------------------------------------------------------------- |
236 | 237 | ||
238 | async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { | ||
239 | const content = await readFile(playlistPath, 'utf8') | ||
240 | |||
241 | const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) | ||
242 | |||
243 | await writeFile(playlistPath, newContent, 'utf8') | ||
244 | } | ||
245 | |||
246 | // --------------------------------------------------------------------------- | ||
247 | |||
237 | function injectQueryToPlaylistUrls (content: string, queryString: string) { | 248 | function injectQueryToPlaylistUrls (content: string, queryString: string) { |
238 | return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) | 249 | return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) |
239 | } | 250 | } |
@@ -247,7 +258,8 @@ export { | |||
247 | downloadPlaylistSegments, | 258 | downloadPlaylistSegments, |
248 | updateStreamingPlaylistsInfohashesIfNeeded, | 259 | updateStreamingPlaylistsInfohashesIfNeeded, |
249 | updatePlaylistAfterFileChange, | 260 | updatePlaylistAfterFileChange, |
250 | injectQueryToPlaylistUrls | 261 | injectQueryToPlaylistUrls, |
262 | renameVideoFileInPlaylist | ||
251 | } | 263 | } |
252 | 264 | ||
253 | // --------------------------------------------------------------------------- | 265 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/lib/job-queue/handlers/transcoding-job-builder.ts new file mode 100644 index 000000000..8b4a877d7 --- /dev/null +++ b/server/lib/job-queue/handlers/transcoding-job-builder.ts | |||
@@ -0,0 +1,47 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | ||
3 | import { UserModel } from '@server/models/user/user' | ||
4 | import { VideoModel } from '@server/models/video/video' | ||
5 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
6 | import { pick } from '@shared/core-utils' | ||
7 | import { TranscodingJobBuilderPayload } from '@shared/models' | ||
8 | import { logger } from '../../../helpers/logger' | ||
9 | import { JobQueue } from '../job-queue' | ||
10 | |||
11 | async function processTranscodingJobBuilder (job: Job) { | ||
12 | const payload = job.data as TranscodingJobBuilderPayload | ||
13 | |||
14 | logger.info('Processing transcoding job builder in job %s.', job.id) | ||
15 | |||
16 | if (payload.optimizeJob) { | ||
17 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
18 | const user = await UserModel.loadByVideoId(video.id) | ||
19 | const videoFile = video.getMaxQualityFile() | ||
20 | |||
21 | await createOptimizeOrMergeAudioJobs({ | ||
22 | ...pick(payload.optimizeJob, [ 'isNewVideo' ]), | ||
23 | |||
24 | video, | ||
25 | videoFile, | ||
26 | user | ||
27 | }) | ||
28 | } | ||
29 | |||
30 | for (const job of (payload.jobs || [])) { | ||
31 | await JobQueue.Instance.createJob(job) | ||
32 | |||
33 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
34 | } | ||
35 | |||
36 | for (const sequentialJobs of (payload.sequentialJobs || [])) { | ||
37 | await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs) | ||
38 | |||
39 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.length) | ||
40 | } | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | export { | ||
46 | processTranscodingJobBuilder | ||
47 | } | ||
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index d950f6407..9a4550e4d 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -10,8 +10,8 @@ import { VideoModel } from '@server/models/video/video' | |||
10 | import { VideoFileModel } from '@server/models/video/video-file' | 10 | import { VideoFileModel } from '@server/models/video/video-file' |
11 | import { MVideoFullLight } from '@server/types/models' | 11 | import { MVideoFullLight } from '@server/types/models' |
12 | import { getLowercaseExtension } from '@shared/core-utils' | 12 | import { getLowercaseExtension } from '@shared/core-utils' |
13 | import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' | ||
13 | import { VideoFileImportPayload, VideoStorage } from '@shared/models' | 14 | import { VideoFileImportPayload, VideoStorage } from '@shared/models' |
14 | import { getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg' | ||
15 | import { logger } from '../../../helpers/logger' | 15 | import { logger } from '../../../helpers/logger' |
16 | import { JobQueue } from '../job-queue' | 16 | import { JobQueue } from '../job-queue' |
17 | 17 | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 4d361c7b9..2a063282c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -7,15 +7,16 @@ import { isPostImportVideoAccepted } from '@server/lib/moderation' | |||
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | 7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' |
8 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
9 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
10 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | ||
10 | import { isAbleToUploadVideo } from '@server/lib/user' | 11 | import { isAbleToUploadVideo } from '@server/lib/user' |
11 | import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video' | 12 | import { buildMoveToObjectStorageJob } from '@server/lib/video' |
12 | import { VideoPathManager } from '@server/lib/video-path-manager' | 13 | import { VideoPathManager } from '@server/lib/video-path-manager' |
13 | import { buildNextVideoState } from '@server/lib/video-state' | 14 | import { buildNextVideoState } from '@server/lib/video-state' |
14 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 15 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
15 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | 16 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' |
16 | import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' | 17 | import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' |
17 | import { getLowercaseExtension } from '@shared/core-utils' | 18 | import { getLowercaseExtension } from '@shared/core-utils' |
18 | import { isAudioFile } from '@shared/extra-utils' | 19 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' |
19 | import { | 20 | import { |
20 | ThumbnailType, | 21 | ThumbnailType, |
21 | VideoImportPayload, | 22 | VideoImportPayload, |
@@ -28,7 +29,6 @@ import { | |||
28 | VideoResolution, | 29 | VideoResolution, |
29 | VideoState | 30 | VideoState |
30 | } from '@shared/models' | 31 | } from '@shared/models' |
31 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg' | ||
32 | import { logger } from '../../../helpers/logger' | 32 | import { logger } from '../../../helpers/logger' |
33 | import { getSecureTorrentName } from '../../../helpers/utils' | 33 | import { getSecureTorrentName } from '../../../helpers/utils' |
34 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' | 34 | import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' |
@@ -137,7 +137,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
137 | 137 | ||
138 | const { resolution } = await isAudioFile(tempVideoPath, probe) | 138 | const { resolution } = await isAudioFile(tempVideoPath, probe) |
139 | ? { resolution: VideoResolution.H_NOVIDEO } | 139 | ? { resolution: VideoResolution.H_NOVIDEO } |
140 | : await getVideoStreamDimensionsInfo(tempVideoPath) | 140 | : await getVideoStreamDimensionsInfo(tempVideoPath, probe) |
141 | 141 | ||
142 | const fps = await getVideoStreamFPS(tempVideoPath, probe) | 142 | const fps = await getVideoStreamFPS(tempVideoPath, probe) |
143 | const duration = await getVideoStreamDuration(tempVideoPath, probe) | 143 | const duration = await getVideoStreamDuration(tempVideoPath, probe) |
@@ -313,9 +313,7 @@ async function afterImportSuccess (options: { | |||
313 | } | 313 | } |
314 | 314 | ||
315 | if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? | 315 | if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? |
316 | await JobQueue.Instance.createJob( | 316 | await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user }) |
317 | await buildOptimizeOrMergeAudioJob({ video, videoFile, user }) | ||
318 | ) | ||
319 | } | 317 | } |
320 | } | 318 | } |
321 | 319 | ||
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 2f3a971bd..1bf43f592 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -1,25 +1,25 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { readdir, remove } from 'fs-extra' | 2 | import { readdir, remove } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo } from '@server/helpers/ffmpeg' | ||
5 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 4 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
6 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | 5 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' |
7 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' | 6 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' |
8 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' | 7 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' |
9 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 8 | import { generateVideoMiniature } from '@server/lib/thumbnail' |
10 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/transcoding' | 9 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' |
10 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
11 | import { moveToNextState } from '@server/lib/video-state' | 11 | import { moveToNextState } from '@server/lib/video-state' |
12 | import { VideoModel } from '@server/models/video/video' | 12 | import { VideoModel } from '@server/models/video/video' |
13 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' | 13 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' |
14 | import { VideoFileModel } from '@server/models/video/video-file' | 14 | import { VideoFileModel } from '@server/models/video/video-file' |
15 | import { VideoLiveModel } from '@server/models/video/video-live' | 15 | import { VideoLiveModel } from '@server/models/video/video-live' |
16 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
16 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | 17 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' |
17 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 18 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
18 | import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' | 19 | import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' |
20 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' | ||
19 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' | 21 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' |
20 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 22 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
21 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
22 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
23 | 23 | ||
24 | const lTags = loggerTagsFactory('live', 'job') | 24 | const lTags = loggerTagsFactory('live', 'job') |
25 | 25 | ||
@@ -224,6 +224,7 @@ async function assignReplayFilesToVideo (options: { | |||
224 | const probe = await ffprobePromise(concatenatedTsFilePath) | 224 | const probe = await ffprobePromise(concatenatedTsFilePath) |
225 | const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) | 225 | const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) |
226 | const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) | 226 | const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) |
227 | const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe) | ||
227 | 228 | ||
228 | try { | 229 | try { |
229 | await generateHlsPlaylistResolutionFromTS({ | 230 | await generateHlsPlaylistResolutionFromTS({ |
@@ -231,6 +232,7 @@ async function assignReplayFilesToVideo (options: { | |||
231 | inputFileMutexReleaser, | 232 | inputFileMutexReleaser, |
232 | concatenatedTsFilePath, | 233 | concatenatedTsFilePath, |
233 | resolution, | 234 | resolution, |
235 | fps, | ||
234 | isAAC: audioStream?.codec_name === 'aac' | 236 | isAAC: audioStream?.codec_name === 'aac' |
235 | }) | 237 | }) |
236 | } catch (err) { | 238 | } catch (err) { |
diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts index 3e208d83d..991d11ef1 100644 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ b/server/lib/job-queue/handlers/video-studio-edition.ts | |||
@@ -1,15 +1,16 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { move, remove } from 'fs-extra' | 2 | import { move, remove } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { addIntroOutro, addWatermark, cutVideo } from '@server/helpers/ffmpeg' | 4 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' |
5 | import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' | 5 | import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' |
6 | import { CONFIG } from '@server/initializers/config' | 6 | import { CONFIG } from '@server/initializers/config' |
7 | import { VIDEO_FILTERS } from '@server/initializers/constants' | ||
7 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | 8 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' |
8 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | 9 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' |
10 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | ||
9 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' | 11 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' |
10 | import { isAbleToUploadVideo } from '@server/lib/user' | 12 | import { isAbleToUploadVideo } from '@server/lib/user' |
11 | import { buildOptimizeOrMergeAudioJob } from '@server/lib/video' | 13 | import { buildFileMetadata, removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' |
12 | import { removeHLSPlaylist, removeWebTorrentFile } from '@server/lib/video-file' | ||
13 | import { VideoPathManager } from '@server/lib/video-path-manager' | 14 | import { VideoPathManager } from '@server/lib/video-path-manager' |
14 | import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' | 15 | import { approximateIntroOutroAdditionalSize } from '@server/lib/video-studio' |
15 | import { UserModel } from '@server/models/user/user' | 16 | import { UserModel } from '@server/models/user/user' |
@@ -17,15 +18,8 @@ import { VideoModel } from '@server/models/video/video' | |||
17 | import { VideoFileModel } from '@server/models/video/video-file' | 18 | import { VideoFileModel } from '@server/models/video/video-file' |
18 | import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models' | 19 | import { MVideo, MVideoFile, MVideoFullLight, MVideoId, MVideoWithAllFiles } from '@server/types/models' |
19 | import { getLowercaseExtension, pick } from '@shared/core-utils' | 20 | import { getLowercaseExtension, pick } from '@shared/core-utils' |
20 | import { | 21 | import { buildUUID, getFileSize } from '@shared/extra-utils' |
21 | buildFileMetadata, | 22 | import { FFmpegEdition, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' |
22 | buildUUID, | ||
23 | ffprobePromise, | ||
24 | getFileSize, | ||
25 | getVideoStreamDimensionsInfo, | ||
26 | getVideoStreamDuration, | ||
27 | getVideoStreamFPS | ||
28 | } from '@shared/extra-utils' | ||
29 | import { | 23 | import { |
30 | VideoStudioEditionPayload, | 24 | VideoStudioEditionPayload, |
31 | VideoStudioTask, | 25 | VideoStudioTask, |
@@ -36,7 +30,6 @@ import { | |||
36 | VideoStudioTaskWatermarkPayload | 30 | VideoStudioTaskWatermarkPayload |
37 | } from '@shared/models' | 31 | } from '@shared/models' |
38 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 32 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
39 | import { JobQueue } from '../job-queue' | ||
40 | 33 | ||
41 | const lTagsBase = loggerTagsFactory('video-edition') | 34 | const lTagsBase = loggerTagsFactory('video-edition') |
42 | 35 | ||
@@ -102,9 +95,7 @@ async function processVideoStudioEdition (job: Job) { | |||
102 | 95 | ||
103 | const user = await UserModel.loadByVideoId(video.id) | 96 | const user = await UserModel.loadByVideoId(video.id) |
104 | 97 | ||
105 | await JobQueue.Instance.createJob( | 98 | await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user }) |
106 | await buildOptimizeOrMergeAudioJob({ video, videoFile: newFile, user, isNewVideo: false }) | ||
107 | ) | ||
108 | } | 99 | } |
109 | 100 | ||
110 | // --------------------------------------------------------------------------- | 101 | // --------------------------------------------------------------------------- |
@@ -131,9 +122,9 @@ const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessor | |||
131 | } | 122 | } |
132 | 123 | ||
133 | async function processTask (options: TaskProcessorOptions) { | 124 | async function processTask (options: TaskProcessorOptions) { |
134 | const { video, task } = options | 125 | const { video, task, lTags } = options |
135 | 126 | ||
136 | logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...options.lTags }) | 127 | logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags }) |
137 | 128 | ||
138 | const processor = taskProcessors[options.task.name] | 129 | const processor = taskProcessors[options.task.name] |
139 | if (!process) throw new Error('Unknown task ' + task.name) | 130 | if (!process) throw new Error('Unknown task ' + task.name) |
@@ -142,48 +133,53 @@ async function processTask (options: TaskProcessorOptions) { | |||
142 | } | 133 | } |
143 | 134 | ||
144 | function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) { | 135 | function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) { |
145 | const { task } = options | 136 | const { task, lTags } = options |
137 | |||
138 | logger.debug('Will add intro/outro to the video.', { options, ...lTags }) | ||
146 | 139 | ||
147 | return addIntroOutro({ | 140 | return buildFFmpegEdition().addIntroOutro({ |
148 | ...pick(options, [ 'inputPath', 'outputPath' ]), | 141 | ...pick(options, [ 'inputPath', 'outputPath' ]), |
149 | 142 | ||
150 | introOutroPath: task.options.file, | 143 | introOutroPath: task.options.file, |
151 | type: task.name === 'add-intro' | 144 | type: task.name === 'add-intro' |
152 | ? 'intro' | 145 | ? 'intro' |
153 | : 'outro', | 146 | : 'outro' |
154 | |||
155 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
156 | profile: CONFIG.TRANSCODING.PROFILE | ||
157 | }) | 147 | }) |
158 | } | 148 | } |
159 | 149 | ||
160 | function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) { | 150 | function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) { |
161 | const { task } = options | 151 | const { task, lTags } = options |
162 | 152 | ||
163 | return cutVideo({ | 153 | logger.debug('Will cut the video.', { options, ...lTags }) |
154 | |||
155 | return buildFFmpegEdition().cutVideo({ | ||
164 | ...pick(options, [ 'inputPath', 'outputPath' ]), | 156 | ...pick(options, [ 'inputPath', 'outputPath' ]), |
165 | 157 | ||
166 | start: task.options.start, | 158 | start: task.options.start, |
167 | end: task.options.end, | 159 | end: task.options.end |
168 | |||
169 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
170 | profile: CONFIG.TRANSCODING.PROFILE | ||
171 | }) | 160 | }) |
172 | } | 161 | } |
173 | 162 | ||
174 | function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) { | 163 | function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) { |
175 | const { task } = options | 164 | const { task, lTags } = options |
165 | |||
166 | logger.debug('Will add watermark to the video.', { options, ...lTags }) | ||
176 | 167 | ||
177 | return addWatermark({ | 168 | return buildFFmpegEdition().addWatermark({ |
178 | ...pick(options, [ 'inputPath', 'outputPath' ]), | 169 | ...pick(options, [ 'inputPath', 'outputPath' ]), |
179 | 170 | ||
180 | watermarkPath: task.options.file, | 171 | watermarkPath: task.options.file, |
181 | 172 | ||
182 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | 173 | videoFilters: { |
183 | profile: CONFIG.TRANSCODING.PROFILE | 174 | watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, |
175 | horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, | ||
176 | verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO | ||
177 | } | ||
184 | }) | 178 | }) |
185 | } | 179 | } |
186 | 180 | ||
181 | // --------------------------------------------------------------------------- | ||
182 | |||
187 | async function buildNewFile (video: MVideoId, path: string) { | 183 | async function buildNewFile (video: MVideoId, path: string) { |
188 | const videoFile = new VideoFileModel({ | 184 | const videoFile = new VideoFileModel({ |
189 | extname: getLowercaseExtension(path), | 185 | extname: getLowercaseExtension(path), |
@@ -223,3 +219,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud | |||
223 | throw new Error('Quota exceeded for this user to edit the video') | 219 | throw new Error('Quota exceeded for this user to edit the video') |
224 | } | 220 | } |
225 | } | 221 | } |
222 | |||
223 | function buildFFmpegEdition () { | ||
224 | return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) | ||
225 | } | ||
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index 3e6d23363..17b717275 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -1,13 +1,13 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { TranscodeVODOptionsType } from '@server/helpers/ffmpeg' | 2 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' |
3 | import { Hooks } from '@server/lib/plugins/hooks' | 3 | import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' |
4 | import { buildTranscodingJob, getTranscodingJobPriority } from '@server/lib/video' | 4 | import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebTorrentResolution } from '@server/lib/transcoding/web-transcoding' |
5 | import { removeAllWebTorrentFiles } from '@server/lib/video-file' | ||
5 | import { VideoPathManager } from '@server/lib/video-path-manager' | 6 | import { VideoPathManager } from '@server/lib/video-path-manager' |
6 | import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' | 7 | import { moveToFailedTranscodingState } from '@server/lib/video-state' |
7 | import { UserModel } from '@server/models/user/user' | 8 | import { UserModel } from '@server/models/user/user' |
8 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
9 | import { MUser, MUserId, MVideo, MVideoFullLight, MVideoWithFile } from '@server/types/models' | 10 | import { MUser, MUserId, MVideoFullLight } from '@server/types/models' |
10 | import { pick } from '@shared/core-utils' | ||
11 | import { | 11 | import { |
12 | HLSTranscodingPayload, | 12 | HLSTranscodingPayload, |
13 | MergeAudioTranscodingPayload, | 13 | MergeAudioTranscodingPayload, |
@@ -15,18 +15,8 @@ import { | |||
15 | OptimizeTranscodingPayload, | 15 | OptimizeTranscodingPayload, |
16 | VideoTranscodingPayload | 16 | VideoTranscodingPayload |
17 | } from '@shared/models' | 17 | } from '@shared/models' |
18 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | ||
19 | import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg' | ||
20 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 18 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
21 | import { CONFIG } from '../../../initializers/config' | ||
22 | import { VideoModel } from '../../../models/video/video' | 19 | import { VideoModel } from '../../../models/video/video' |
23 | import { | ||
24 | generateHlsPlaylistResolution, | ||
25 | mergeAudioVideofile, | ||
26 | optimizeOriginalVideofile, | ||
27 | transcodeNewWebTorrentResolution | ||
28 | } from '../../transcoding/transcoding' | ||
29 | import { JobQueue } from '../job-queue' | ||
30 | 20 | ||
31 | type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void> | 21 | type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void> |
32 | 22 | ||
@@ -84,260 +74,72 @@ export { | |||
84 | // Job handlers | 74 | // Job handlers |
85 | // --------------------------------------------------------------------------- | 75 | // --------------------------------------------------------------------------- |
86 | 76 | ||
87 | async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight, user: MUser) { | ||
88 | logger.info('Handling HLS transcoding job for %s.', video.uuid, lTags(video.uuid)) | ||
89 | |||
90 | const videoFileInput = payload.copyCodecs | ||
91 | ? video.getWebTorrentFile(payload.resolution) | ||
92 | : video.getMaxQualityFile() | ||
93 | |||
94 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | ||
95 | |||
96 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
97 | |||
98 | try { | ||
99 | await videoFileInput.getVideo().reload() | ||
100 | |||
101 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { | ||
102 | return generateHlsPlaylistResolution({ | ||
103 | video, | ||
104 | videoInputPath, | ||
105 | inputFileMutexReleaser, | ||
106 | resolution: payload.resolution, | ||
107 | copyCodecs: payload.copyCodecs, | ||
108 | job | ||
109 | }) | ||
110 | }) | ||
111 | } finally { | ||
112 | inputFileMutexReleaser() | ||
113 | } | ||
114 | |||
115 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) | ||
116 | |||
117 | await onHlsPlaylistGeneration(video, user, payload) | ||
118 | } | ||
119 | |||
120 | async function handleNewWebTorrentResolutionJob ( | ||
121 | job: Job, | ||
122 | payload: NewWebTorrentResolutionTranscodingPayload, | ||
123 | video: MVideoFullLight, | ||
124 | user: MUserId | ||
125 | ) { | ||
126 | logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid)) | ||
127 | |||
128 | await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, job }) | ||
129 | |||
130 | logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid)) | ||
131 | |||
132 | await onNewWebTorrentFileResolution(video, user, payload) | ||
133 | } | ||
134 | |||
135 | async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 77 | async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
136 | logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid)) | 78 | logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid)) |
137 | 79 | ||
138 | await mergeAudioVideofile({ video, resolution: payload.resolution, job }) | 80 | await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) |
139 | 81 | ||
140 | logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid)) | 82 | logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid)) |
141 | 83 | ||
142 | await onVideoFirstWebTorrentTranscoding(video, payload, 'video', user) | 84 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) |
143 | } | 85 | } |
144 | 86 | ||
145 | async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 87 | async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
146 | logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid)) | 88 | logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid)) |
147 | 89 | ||
148 | const { transcodeType } = await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), job }) | 90 | await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) |
149 | 91 | ||
150 | logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid)) | 92 | logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid)) |
151 | 93 | ||
152 | await onVideoFirstWebTorrentTranscoding(video, payload, transcodeType, user) | 94 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) |
153 | } | 95 | } |
154 | 96 | ||
155 | // --------------------------------------------------------------------------- | 97 | async function handleNewWebTorrentResolutionJob (job: Job, payload: NewWebTorrentResolutionTranscodingPayload, video: MVideoFullLight) { |
156 | 98 | logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid)) | |
157 | async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, payload: HLSTranscodingPayload) { | ||
158 | if (payload.isMaxQuality && payload.autoDeleteWebTorrentIfNeeded && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { | ||
159 | // Remove webtorrent files if not enabled | ||
160 | for (const file of video.VideoFiles) { | ||
161 | await video.removeWebTorrentFile(file) | ||
162 | await file.destroy() | ||
163 | } | ||
164 | |||
165 | video.VideoFiles = [] | ||
166 | |||
167 | // Create HLS new resolution jobs | ||
168 | await createLowerResolutionsJobs({ | ||
169 | video, | ||
170 | user, | ||
171 | videoFileResolution: payload.resolution, | ||
172 | hasAudio: payload.hasAudio, | ||
173 | isNewVideo: payload.isNewVideo ?? true, | ||
174 | type: 'hls' | ||
175 | }) | ||
176 | } | ||
177 | |||
178 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
179 | await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo }) | ||
180 | } | ||
181 | 99 | ||
182 | async function onVideoFirstWebTorrentTranscoding ( | 100 | await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) |
183 | videoArg: MVideoWithFile, | ||
184 | payload: OptimizeTranscodingPayload | MergeAudioTranscodingPayload, | ||
185 | transcodeType: TranscodeVODOptionsType, | ||
186 | user: MUserId | ||
187 | ) { | ||
188 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) | ||
189 | 101 | ||
190 | try { | 102 | logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid)) |
191 | // Maybe the video changed in database, refresh it | ||
192 | const videoDatabase = await VideoModel.loadFull(videoArg.uuid) | ||
193 | // Video does not exist anymore | ||
194 | if (!videoDatabase) return undefined | ||
195 | |||
196 | const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile() | ||
197 | |||
198 | // Generate HLS version of the original file | ||
199 | const originalFileHLSPayload = { | ||
200 | ...payload, | ||
201 | |||
202 | hasAudio: !!audioStream, | ||
203 | resolution: videoDatabase.getMaxQualityFile().resolution, | ||
204 | // If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues | ||
205 | copyCodecs: transcodeType !== 'quick-transcode', | ||
206 | isMaxQuality: true | ||
207 | } | ||
208 | const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload) | ||
209 | const hasNewResolutions = await createLowerResolutionsJobs({ | ||
210 | video: videoDatabase, | ||
211 | user, | ||
212 | videoFileResolution: resolution, | ||
213 | hasAudio: !!audioStream, | ||
214 | type: 'webtorrent', | ||
215 | isNewVideo: payload.isNewVideo ?? true | ||
216 | }) | ||
217 | |||
218 | await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode') | ||
219 | 103 | ||
220 | // Move to next state if there are no other resolutions to generate | 104 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) |
221 | if (!hasHls && !hasNewResolutions) { | ||
222 | await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo }) | ||
223 | } | ||
224 | } finally { | ||
225 | mutexReleaser() | ||
226 | } | ||
227 | } | 105 | } |
228 | 106 | ||
229 | async function onNewWebTorrentFileResolution ( | 107 | async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MVideoFullLight) { |
230 | video: MVideo, | 108 | logger.info('Handling HLS transcoding job for %s.', video.uuid, lTags(video.uuid)) |
231 | user: MUserId, | ||
232 | payload: NewWebTorrentResolutionTranscodingPayload | MergeAudioTranscodingPayload | ||
233 | ) { | ||
234 | if (payload.createHLSIfNeeded) { | ||
235 | await createHlsJobIfEnabled(user, { hasAudio: true, copyCodecs: true, isMaxQuality: false, ...payload }) | ||
236 | } | ||
237 | |||
238 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
239 | 109 | ||
240 | await retryTransactionWrapper(moveToNextState, { video, isNewVideo: payload.isNewVideo }) | 110 | const videoFileInput = payload.copyCodecs |
241 | } | 111 | ? video.getWebTorrentFile(payload.resolution) |
112 | : video.getMaxQualityFile() | ||
242 | 113 | ||
243 | // --------------------------------------------------------------------------- | 114 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() |
244 | 115 | ||
245 | async function createHlsJobIfEnabled (user: MUserId, payload: { | 116 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
246 | videoUUID: string | ||
247 | resolution: number | ||
248 | hasAudio: boolean | ||
249 | copyCodecs: boolean | ||
250 | isMaxQuality: boolean | ||
251 | isNewVideo?: boolean | ||
252 | }) { | ||
253 | if (!payload || CONFIG.TRANSCODING.ENABLED !== true || CONFIG.TRANSCODING.HLS.ENABLED !== true) return false | ||
254 | |||
255 | const jobOptions = { | ||
256 | priority: await getTranscodingJobPriority(user) | ||
257 | } | ||
258 | 117 | ||
259 | const hlsTranscodingPayload: HLSTranscodingPayload = { | 118 | try { |
260 | type: 'new-resolution-to-hls', | 119 | await videoFileInput.getVideo().reload() |
261 | autoDeleteWebTorrentIfNeeded: true, | ||
262 | 120 | ||
263 | ...pick(payload, [ 'videoUUID', 'resolution', 'copyCodecs', 'isMaxQuality', 'isNewVideo', 'hasAudio' ]) | 121 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { |
122 | return generateHlsPlaylistResolution({ | ||
123 | video, | ||
124 | videoInputPath, | ||
125 | inputFileMutexReleaser, | ||
126 | resolution: payload.resolution, | ||
127 | fps: payload.fps, | ||
128 | copyCodecs: payload.copyCodecs, | ||
129 | job | ||
130 | }) | ||
131 | }) | ||
132 | } finally { | ||
133 | inputFileMutexReleaser() | ||
264 | } | 134 | } |
265 | 135 | ||
266 | await JobQueue.Instance.createJob(await buildTranscodingJob(hlsTranscodingPayload, jobOptions)) | 136 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid)) |
267 | |||
268 | return true | ||
269 | } | ||
270 | |||
271 | async function createLowerResolutionsJobs (options: { | ||
272 | video: MVideoFullLight | ||
273 | user: MUserId | ||
274 | videoFileResolution: number | ||
275 | hasAudio: boolean | ||
276 | isNewVideo: boolean | ||
277 | type: 'hls' | 'webtorrent' | ||
278 | }) { | ||
279 | const { video, user, videoFileResolution, isNewVideo, hasAudio, type } = options | ||
280 | |||
281 | // Create transcoding jobs if there are enabled resolutions | ||
282 | const resolutionsEnabled = await Hooks.wrapObject( | ||
283 | computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), | ||
284 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
285 | options | ||
286 | ) | ||
287 | |||
288 | const resolutionCreated: string[] = [] | ||
289 | |||
290 | for (const resolution of resolutionsEnabled) { | ||
291 | let dataInput: VideoTranscodingPayload | ||
292 | |||
293 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED && type === 'webtorrent') { | ||
294 | // WebTorrent will create subsequent HLS job | ||
295 | dataInput = { | ||
296 | type: 'new-resolution-to-webtorrent', | ||
297 | videoUUID: video.uuid, | ||
298 | resolution, | ||
299 | hasAudio, | ||
300 | createHLSIfNeeded: true, | ||
301 | isNewVideo | ||
302 | } | ||
303 | |||
304 | resolutionCreated.push('webtorrent-' + resolution) | ||
305 | } | ||
306 | |||
307 | if (CONFIG.TRANSCODING.HLS.ENABLED && type === 'hls') { | ||
308 | dataInput = { | ||
309 | type: 'new-resolution-to-hls', | ||
310 | videoUUID: video.uuid, | ||
311 | resolution, | ||
312 | hasAudio, | ||
313 | copyCodecs: false, | ||
314 | isMaxQuality: false, | ||
315 | autoDeleteWebTorrentIfNeeded: true, | ||
316 | isNewVideo | ||
317 | } | ||
318 | |||
319 | resolutionCreated.push('hls-' + resolution) | ||
320 | } | ||
321 | |||
322 | if (!dataInput) continue | ||
323 | |||
324 | const jobOptions = { | ||
325 | priority: await getTranscodingJobPriority(user) | ||
326 | } | ||
327 | |||
328 | await JobQueue.Instance.createJob(await buildTranscodingJob(dataInput, jobOptions)) | ||
329 | } | ||
330 | 137 | ||
331 | if (resolutionCreated.length === 0) { | 138 | if (payload.deleteWebTorrentFiles === true) { |
332 | logger.info('No transcoding jobs created for video %s (no resolutions).', video.uuid, lTags(video.uuid)) | 139 | logger.info('Removing WebTorrent files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) |
333 | 140 | ||
334 | return false | 141 | await removeAllWebTorrentFiles(video) |
335 | } | 142 | } |
336 | 143 | ||
337 | logger.info( | 144 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) |
338 | 'New resolutions %s transcoding jobs created for video %s and origin file resolution of %d.', type, video.uuid, videoFileResolution, | ||
339 | { resolutionCreated, ...lTags(video.uuid) } | ||
340 | ) | ||
341 | |||
342 | return true | ||
343 | } | 145 | } |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index cc6be0bd8..21bf0f226 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -31,6 +31,7 @@ import { | |||
31 | MoveObjectStoragePayload, | 31 | MoveObjectStoragePayload, |
32 | NotifyPayload, | 32 | NotifyPayload, |
33 | RefreshPayload, | 33 | RefreshPayload, |
34 | TranscodingJobBuilderPayload, | ||
34 | VideoChannelImportPayload, | 35 | VideoChannelImportPayload, |
35 | VideoFileImportPayload, | 36 | VideoFileImportPayload, |
36 | VideoImportPayload, | 37 | VideoImportPayload, |
@@ -56,6 +57,7 @@ import { processFederateVideo } from './handlers/federate-video' | |||
56 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' | 57 | import { processManageVideoTorrent } from './handlers/manage-video-torrent' |
57 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' | 58 | import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' |
58 | import { processNotify } from './handlers/notify' | 59 | import { processNotify } from './handlers/notify' |
60 | import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder' | ||
59 | import { processVideoChannelImport } from './handlers/video-channel-import' | 61 | import { processVideoChannelImport } from './handlers/video-channel-import' |
60 | import { processVideoFileImport } from './handlers/video-file-import' | 62 | import { processVideoFileImport } from './handlers/video-file-import' |
61 | import { processVideoImport } from './handlers/video-import' | 63 | import { processVideoImport } from './handlers/video-import' |
@@ -69,11 +71,12 @@ export type CreateJobArgument = | |||
69 | { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | | 71 | { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | |
70 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | 72 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | |
71 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | 73 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | |
72 | { type: 'activitypub-http-cleaner', payload: {} } | | 74 | { type: 'activitypub-cleaner', payload: {} } | |
73 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | 75 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
74 | { type: 'video-file-import', payload: VideoFileImportPayload } | | 76 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
75 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | | 77 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | |
76 | { type: 'email', payload: EmailPayload } | | 78 | { type: 'email', payload: EmailPayload } | |
79 | { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } | | ||
77 | { type: 'video-import', payload: VideoImportPayload } | | 80 | { type: 'video-import', payload: VideoImportPayload } | |
78 | { type: 'activitypub-refresher', payload: RefreshPayload } | | 81 | { type: 'activitypub-refresher', payload: RefreshPayload } | |
79 | { type: 'videos-views-stats', payload: {} } | | 82 | { type: 'videos-views-stats', payload: {} } | |
@@ -96,28 +99,29 @@ export type CreateJobOptions = { | |||
96 | } | 99 | } |
97 | 100 | ||
98 | const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { | 101 | const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { |
99 | 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, | ||
100 | 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, | ||
101 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | ||
102 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | ||
103 | 'activitypub-cleaner': processActivityPubCleaner, | 102 | 'activitypub-cleaner': processActivityPubCleaner, |
104 | 'activitypub-follow': processActivityPubFollow, | 103 | 'activitypub-follow': processActivityPubFollow, |
105 | 'video-file-import': processVideoFileImport, | 104 | 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, |
106 | 'video-transcoding': processVideoTranscoding, | 105 | 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, |
106 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | ||
107 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | ||
108 | 'activitypub-refresher': refreshAPObject, | ||
109 | 'actor-keys': processActorKeys, | ||
110 | 'after-video-channel-import': processAfterVideoChannelImport, | ||
107 | 'email': processEmail, | 111 | 'email': processEmail, |
112 | 'federate-video': processFederateVideo, | ||
113 | 'transcoding-job-builder': processTranscodingJobBuilder, | ||
114 | 'manage-video-torrent': processManageVideoTorrent, | ||
115 | 'move-to-object-storage': processMoveToObjectStorage, | ||
116 | 'notify': processNotify, | ||
117 | 'video-channel-import': processVideoChannelImport, | ||
118 | 'video-file-import': processVideoFileImport, | ||
108 | 'video-import': processVideoImport, | 119 | 'video-import': processVideoImport, |
109 | 'videos-views-stats': processVideosViewsStats, | ||
110 | 'activitypub-refresher': refreshAPObject, | ||
111 | 'video-live-ending': processVideoLiveEnding, | 120 | 'video-live-ending': processVideoLiveEnding, |
112 | 'actor-keys': processActorKeys, | ||
113 | 'video-redundancy': processVideoRedundancy, | 121 | 'video-redundancy': processVideoRedundancy, |
114 | 'move-to-object-storage': processMoveToObjectStorage, | ||
115 | 'manage-video-torrent': processManageVideoTorrent, | ||
116 | 'video-studio-edition': processVideoStudioEdition, | 122 | 'video-studio-edition': processVideoStudioEdition, |
117 | 'video-channel-import': processVideoChannelImport, | 123 | 'video-transcoding': processVideoTranscoding, |
118 | 'after-video-channel-import': processAfterVideoChannelImport, | 124 | 'videos-views-stats': processVideosViewsStats |
119 | 'notify': processNotify, | ||
120 | 'federate-video': processFederateVideo | ||
121 | } | 125 | } |
122 | 126 | ||
123 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { | 127 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { |
@@ -125,28 +129,29 @@ const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } | |||
125 | } | 129 | } |
126 | 130 | ||
127 | const jobTypes: JobType[] = [ | 131 | const jobTypes: JobType[] = [ |
132 | 'activitypub-cleaner', | ||
128 | 'activitypub-follow', | 133 | 'activitypub-follow', |
129 | 'activitypub-http-broadcast', | ||
130 | 'activitypub-http-broadcast-parallel', | 134 | 'activitypub-http-broadcast-parallel', |
135 | 'activitypub-http-broadcast', | ||
131 | 'activitypub-http-fetcher', | 136 | 'activitypub-http-fetcher', |
132 | 'activitypub-http-unicast', | 137 | 'activitypub-http-unicast', |
133 | 'activitypub-cleaner', | 138 | 'activitypub-refresher', |
139 | 'actor-keys', | ||
140 | 'after-video-channel-import', | ||
134 | 'email', | 141 | 'email', |
135 | 'video-transcoding', | 142 | 'federate-video', |
143 | 'transcoding-job-builder', | ||
144 | 'manage-video-torrent', | ||
145 | 'move-to-object-storage', | ||
146 | 'notify', | ||
147 | 'video-channel-import', | ||
136 | 'video-file-import', | 148 | 'video-file-import', |
137 | 'video-import', | 149 | 'video-import', |
138 | 'videos-views-stats', | ||
139 | 'activitypub-refresher', | ||
140 | 'video-redundancy', | ||
141 | 'actor-keys', | ||
142 | 'video-live-ending', | 150 | 'video-live-ending', |
143 | 'move-to-object-storage', | 151 | 'video-redundancy', |
144 | 'manage-video-torrent', | ||
145 | 'video-studio-edition', | 152 | 'video-studio-edition', |
146 | 'video-channel-import', | 153 | 'video-transcoding', |
147 | 'after-video-channel-import', | 154 | 'videos-views-stats' |
148 | 'notify', | ||
149 | 'federate-video' | ||
150 | ] | 155 | ] |
151 | 156 | ||
152 | const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ]) | 157 | const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ]) |
diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 05274955d..aa32a9d52 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts | |||
@@ -2,36 +2,30 @@ import { readdir, readFile } from 'fs-extra' | |||
2 | import { createServer, Server } from 'net' | 2 | import { createServer, Server } from 'net' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { createServer as createServerTLS, Server as ServerTLS } from 'tls' | 4 | import { createServer as createServerTLS, Server as ServerTLS } from 'tls' |
5 | import { | ||
6 | computeResolutionsToTranscode, | ||
7 | ffprobePromise, | ||
8 | getLiveSegmentTime, | ||
9 | getVideoStreamBitrate, | ||
10 | getVideoStreamDimensionsInfo, | ||
11 | getVideoStreamFPS, | ||
12 | hasAudioStream | ||
13 | } from '@server/helpers/ffmpeg' | ||
14 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
15 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' | 6 | import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' |
16 | import { VIDEO_LIVE } from '@server/initializers/constants' | 7 | import { VIDEO_LIVE } from '@server/initializers/constants' |
8 | import { sequelizeTypescript } from '@server/initializers/database' | ||
17 | import { UserModel } from '@server/models/user/user' | 9 | import { UserModel } from '@server/models/user/user' |
18 | import { VideoModel } from '@server/models/video/video' | 10 | import { VideoModel } from '@server/models/video/video' |
19 | import { VideoLiveModel } from '@server/models/video/video-live' | 11 | import { VideoLiveModel } from '@server/models/video/video-live' |
12 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
20 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' | 13 | import { VideoLiveSessionModel } from '@server/models/video/video-live-session' |
21 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' | 14 | import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' |
22 | import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models' | 15 | import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models' |
23 | import { pick, wait } from '@shared/core-utils' | 16 | import { pick, wait } from '@shared/core-utils' |
17 | import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' | ||
24 | import { LiveVideoError, VideoState } from '@shared/models' | 18 | import { LiveVideoError, VideoState } from '@shared/models' |
25 | import { federateVideoIfNeeded } from '../activitypub/videos' | 19 | import { federateVideoIfNeeded } from '../activitypub/videos' |
26 | import { JobQueue } from '../job-queue' | 20 | import { JobQueue } from '../job-queue' |
27 | import { getLiveReplayBaseDirectory } from '../paths' | 21 | import { getLiveReplayBaseDirectory } from '../paths' |
28 | import { PeerTubeSocket } from '../peertube-socket' | 22 | import { PeerTubeSocket } from '../peertube-socket' |
29 | import { Hooks } from '../plugins/hooks' | 23 | import { Hooks } from '../plugins/hooks' |
24 | import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions' | ||
30 | import { LiveQuotaStore } from './live-quota-store' | 25 | import { LiveQuotaStore } from './live-quota-store' |
31 | import { cleanupAndDestroyPermanentLive } from './live-utils' | 26 | import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils' |
32 | import { MuxingSession } from './shared' | 27 | import { MuxingSession } from './shared' |
33 | import { sequelizeTypescript } from '@server/initializers/database' | 28 | import { RunnerJobModel } from '@server/models/runner/runner-job' |
34 | import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' | ||
35 | 29 | ||
36 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') | 30 | const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') |
37 | const context = require('node-media-server/src/node_core_ctx') | 31 | const context = require('node-media-server/src/node_core_ctx') |
@@ -57,7 +51,7 @@ class LiveManager { | |||
57 | private static instance: LiveManager | 51 | private static instance: LiveManager |
58 | 52 | ||
59 | private readonly muxingSessions = new Map<string, MuxingSession>() | 53 | private readonly muxingSessions = new Map<string, MuxingSession>() |
60 | private readonly videoSessions = new Map<number, string>() | 54 | private readonly videoSessions = new Map<string, string>() |
61 | 55 | ||
62 | private rtmpServer: Server | 56 | private rtmpServer: Server |
63 | private rtmpsServer: ServerTLS | 57 | private rtmpsServer: ServerTLS |
@@ -177,14 +171,19 @@ class LiveManager { | |||
177 | return !!this.rtmpServer | 171 | return !!this.rtmpServer |
178 | } | 172 | } |
179 | 173 | ||
180 | stopSessionOf (videoId: number, error: LiveVideoError | null) { | 174 | stopSessionOf (videoUUID: string, error: LiveVideoError | null) { |
181 | const sessionId = this.videoSessions.get(videoId) | 175 | const sessionId = this.videoSessions.get(videoUUID) |
182 | if (!sessionId) return | 176 | if (!sessionId) { |
177 | logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID)) | ||
178 | return | ||
179 | } | ||
183 | 180 | ||
184 | this.saveEndingSession(videoId, error) | 181 | logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) }) |
185 | .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) | ||
186 | 182 | ||
187 | this.videoSessions.delete(videoId) | 183 | this.saveEndingSession(videoUUID, error) |
184 | .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) })) | ||
185 | |||
186 | this.videoSessions.delete(videoUUID) | ||
188 | this.abortSession(sessionId) | 187 | this.abortSession(sessionId) |
189 | } | 188 | } |
190 | 189 | ||
@@ -221,6 +220,11 @@ class LiveManager { | |||
221 | return this.abortSession(sessionId) | 220 | return this.abortSession(sessionId) |
222 | } | 221 | } |
223 | 222 | ||
223 | if (this.videoSessions.has(video.uuid)) { | ||
224 | logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid)) | ||
225 | return this.abortSession(sessionId) | ||
226 | } | ||
227 | |||
224 | // Cleanup old potential live (could happen with a permanent live) | 228 | // Cleanup old potential live (could happen with a permanent live) |
225 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) | 229 | const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) |
226 | if (oldStreamingPlaylist) { | 230 | if (oldStreamingPlaylist) { |
@@ -229,7 +233,7 @@ class LiveManager { | |||
229 | await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) | 233 | await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) |
230 | } | 234 | } |
231 | 235 | ||
232 | this.videoSessions.set(video.id, sessionId) | 236 | this.videoSessions.set(video.uuid, sessionId) |
233 | 237 | ||
234 | const now = Date.now() | 238 | const now = Date.now() |
235 | const probe = await ffprobePromise(inputUrl) | 239 | const probe = await ffprobePromise(inputUrl) |
@@ -253,7 +257,7 @@ class LiveManager { | |||
253 | ) | 257 | ) |
254 | 258 | ||
255 | logger.info( | 259 | logger.info( |
256 | 'Will mux/transcode live video of original resolution %d.', resolution, | 260 | 'Handling live video of original resolution %d.', resolution, |
257 | { allResolutions, ...lTags(sessionId, video.uuid) } | 261 | { allResolutions, ...lTags(sessionId, video.uuid) } |
258 | ) | 262 | ) |
259 | 263 | ||
@@ -301,44 +305,44 @@ class LiveManager { | |||
301 | 305 | ||
302 | muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) | 306 | muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) |
303 | 307 | ||
304 | muxingSession.on('bad-socket-health', ({ videoId }) => { | 308 | muxingSession.on('bad-socket-health', ({ videoUUID }) => { |
305 | logger.error( | 309 | logger.error( |
306 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + | 310 | 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + |
307 | ' Stopping session of video %s.', videoUUID, | 311 | ' Stopping session of video %s.', videoUUID, |
308 | localLTags | 312 | localLTags |
309 | ) | 313 | ) |
310 | 314 | ||
311 | this.stopSessionOf(videoId, LiveVideoError.BAD_SOCKET_HEALTH) | 315 | this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH) |
312 | }) | 316 | }) |
313 | 317 | ||
314 | muxingSession.on('duration-exceeded', ({ videoId }) => { | 318 | muxingSession.on('duration-exceeded', ({ videoUUID }) => { |
315 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) | 319 | logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) |
316 | 320 | ||
317 | this.stopSessionOf(videoId, LiveVideoError.DURATION_EXCEEDED) | 321 | this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED) |
318 | }) | 322 | }) |
319 | 323 | ||
320 | muxingSession.on('quota-exceeded', ({ videoId }) => { | 324 | muxingSession.on('quota-exceeded', ({ videoUUID }) => { |
321 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) | 325 | logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) |
322 | 326 | ||
323 | this.stopSessionOf(videoId, LiveVideoError.QUOTA_EXCEEDED) | 327 | this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED) |
324 | }) | 328 | }) |
325 | 329 | ||
326 | muxingSession.on('ffmpeg-error', ({ videoId }) => { | 330 | muxingSession.on('transcoding-error', ({ videoUUID }) => { |
327 | this.stopSessionOf(videoId, LiveVideoError.FFMPEG_ERROR) | 331 | this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR) |
328 | }) | 332 | }) |
329 | 333 | ||
330 | muxingSession.on('ffmpeg-end', ({ videoId }) => { | 334 | muxingSession.on('transcoding-end', ({ videoUUID }) => { |
331 | this.onMuxingFFmpegEnd(videoId, sessionId) | 335 | this.onMuxingFFmpegEnd(videoUUID, sessionId) |
332 | }) | 336 | }) |
333 | 337 | ||
334 | muxingSession.on('after-cleanup', ({ videoId }) => { | 338 | muxingSession.on('after-cleanup', ({ videoUUID }) => { |
335 | this.muxingSessions.delete(sessionId) | 339 | this.muxingSessions.delete(sessionId) |
336 | 340 | ||
337 | LiveQuotaStore.Instance.removeLive(user.id, videoLive.id) | 341 | LiveQuotaStore.Instance.removeLive(user.id, videoLive.id) |
338 | 342 | ||
339 | muxingSession.destroy() | 343 | muxingSession.destroy() |
340 | 344 | ||
341 | return this.onAfterMuxingCleanup({ videoId, liveSession }) | 345 | return this.onAfterMuxingCleanup({ videoUUID, liveSession }) |
342 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) | 346 | .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) |
343 | }) | 347 | }) |
344 | 348 | ||
@@ -379,22 +383,24 @@ class LiveManager { | |||
379 | } | 383 | } |
380 | } | 384 | } |
381 | 385 | ||
382 | private onMuxingFFmpegEnd (videoId: number, sessionId: string) { | 386 | private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) { |
383 | this.videoSessions.delete(videoId) | 387 | this.videoSessions.delete(videoUUID) |
384 | 388 | ||
385 | this.saveEndingSession(videoId, null) | 389 | this.saveEndingSession(videoUUID, null) |
386 | .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) | 390 | .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) |
387 | } | 391 | } |
388 | 392 | ||
389 | private async onAfterMuxingCleanup (options: { | 393 | private async onAfterMuxingCleanup (options: { |
390 | videoId: number | string | 394 | videoUUID: string |
391 | liveSession?: MVideoLiveSession | 395 | liveSession?: MVideoLiveSession |
392 | cleanupNow?: boolean // Default false | 396 | cleanupNow?: boolean // Default false |
393 | }) { | 397 | }) { |
394 | const { videoId, liveSession: liveSessionArg, cleanupNow = false } = options | 398 | const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options |
399 | |||
400 | logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID)) | ||
395 | 401 | ||
396 | try { | 402 | try { |
397 | const fullVideo = await VideoModel.loadFull(videoId) | 403 | const fullVideo = await VideoModel.loadFull(videoUUID) |
398 | if (!fullVideo) return | 404 | if (!fullVideo) return |
399 | 405 | ||
400 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) | 406 | const live = await VideoLiveModel.loadByVideoId(fullVideo.id) |
@@ -437,15 +443,17 @@ class LiveManager { | |||
437 | 443 | ||
438 | await federateVideoIfNeeded(fullVideo, false) | 444 | await federateVideoIfNeeded(fullVideo, false) |
439 | } catch (err) { | 445 | } catch (err) { |
440 | logger.error('Cannot save/federate new video state of live streaming of video %d.', videoId, { err, ...lTags(videoId + '') }) | 446 | logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) }) |
441 | } | 447 | } |
442 | } | 448 | } |
443 | 449 | ||
444 | private async handleBrokenLives () { | 450 | private async handleBrokenLives () { |
451 | await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' }) | ||
452 | |||
445 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() | 453 | const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() |
446 | 454 | ||
447 | for (const uuid of videoUUIDs) { | 455 | for (const uuid of videoUUIDs) { |
448 | await this.onAfterMuxingCleanup({ videoId: uuid, cleanupNow: true }) | 456 | await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true }) |
449 | } | 457 | } |
450 | } | 458 | } |
451 | 459 | ||
@@ -494,8 +502,8 @@ class LiveManager { | |||
494 | }) | 502 | }) |
495 | } | 503 | } |
496 | 504 | ||
497 | private async saveEndingSession (videoId: number, error: LiveVideoError | null) { | 505 | private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) { |
498 | const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoId) | 506 | const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID) |
499 | if (!liveSession) return | 507 | if (!liveSession) return |
500 | 508 | ||
501 | liveSession.endDate = new Date() | 509 | liveSession.endDate = new Date() |
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts index 4d03754a9..251301141 100644 --- a/server/lib/live/live-segment-sha-store.ts +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -52,7 +52,10 @@ class LiveSegmentShaStore { | |||
52 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) | 52 | logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) |
53 | 53 | ||
54 | if (!this.segmentsSha256.has(segmentName)) { | 54 | if (!this.segmentsSha256.has(segmentName)) { |
55 | logger.warn('Unknown segment in files map for video %s and segment %s.', this.videoUUID, segmentPath, lTags(this.videoUUID)) | 55 | logger.warn( |
56 | 'Unknown segment in live segment hash store for video %s and segment %s.', | ||
57 | this.videoUUID, segmentPath, lTags(this.videoUUID) | ||
58 | ) | ||
56 | return | 59 | return |
57 | } | 60 | } |
58 | 61 | ||
diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts index c0dec9829..3fb3ce1ce 100644 --- a/server/lib/live/live-utils.ts +++ b/server/lib/live/live-utils.ts | |||
@@ -1,8 +1,9 @@ | |||
1 | import { pathExists, readdir, remove } from 'fs-extra' | 1 | import { pathExists, readdir, remove } from 'fs-extra' |
2 | import { basename, join } from 'path' | 2 | import { basename, join } from 'path' |
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { VIDEO_LIVE } from '@server/initializers/constants' | ||
4 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' | 5 | import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' |
5 | import { VideoStorage } from '@shared/models' | 6 | import { LiveVideoLatencyMode, VideoStorage } from '@shared/models' |
6 | import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' | 7 | import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' |
7 | import { getLiveDirectory } from '../paths' | 8 | import { getLiveDirectory } from '../paths' |
8 | 9 | ||
@@ -37,10 +38,19 @@ async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreaming | |||
37 | await cleanupTMPLiveFilesFromFilesystem(video) | 38 | await cleanupTMPLiveFilesFromFilesystem(video) |
38 | } | 39 | } |
39 | 40 | ||
41 | function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { | ||
42 | if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { | ||
43 | return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY | ||
44 | } | ||
45 | |||
46 | return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY | ||
47 | } | ||
48 | |||
40 | export { | 49 | export { |
41 | cleanupAndDestroyPermanentLive, | 50 | cleanupAndDestroyPermanentLive, |
42 | cleanupUnsavedNormalLive, | 51 | cleanupUnsavedNormalLive, |
43 | cleanupTMPLiveFiles, | 52 | cleanupTMPLiveFiles, |
53 | getLiveSegmentTime, | ||
44 | buildConcatenatedName | 54 | buildConcatenatedName |
45 | } | 55 | } |
46 | 56 | ||
diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index 2727fc4a7..f3f8fc886 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts | |||
@@ -1,11 +1,10 @@ | |||
1 | import { mapSeries } from 'bluebird' | 1 | import { mapSeries } from 'bluebird' |
2 | import { FSWatcher, watch } from 'chokidar' | 2 | import { FSWatcher, watch } from 'chokidar' |
3 | import { FfmpegCommand } from 'fluent-ffmpeg' | 3 | import { EventEmitter } from 'events' |
4 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' | 4 | import { appendFile, ensureDir, readFile, stat } from 'fs-extra' |
5 | import PQueue from 'p-queue' | 5 | import PQueue from 'p-queue' |
6 | import { basename, join } from 'path' | 6 | import { basename, join } from 'path' |
7 | import { EventEmitter } from 'stream' | 7 | import { computeOutputFPS } from '@server/helpers/ffmpeg' |
8 | import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg' | ||
9 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' | 8 | import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' |
10 | import { CONFIG } from '@server/initializers/config' | 9 | import { CONFIG } from '@server/initializers/config' |
11 | import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' | 10 | import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' |
@@ -20,24 +19,24 @@ import { | |||
20 | getLiveDirectory, | 19 | getLiveDirectory, |
21 | getLiveReplayBaseDirectory | 20 | getLiveReplayBaseDirectory |
22 | } from '../../paths' | 21 | } from '../../paths' |
23 | import { VideoTranscodingProfilesManager } from '../../transcoding/default-transcoding-profiles' | ||
24 | import { isAbleToUploadVideo } from '../../user' | 22 | import { isAbleToUploadVideo } from '../../user' |
25 | import { LiveQuotaStore } from '../live-quota-store' | 23 | import { LiveQuotaStore } from '../live-quota-store' |
26 | import { LiveSegmentShaStore } from '../live-segment-sha-store' | 24 | import { LiveSegmentShaStore } from '../live-segment-sha-store' |
27 | import { buildConcatenatedName } from '../live-utils' | 25 | import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils' |
26 | import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper' | ||
28 | 27 | ||
29 | import memoizee = require('memoizee') | 28 | import memoizee = require('memoizee') |
30 | interface MuxingSessionEvents { | 29 | interface MuxingSessionEvents { |
31 | 'live-ready': (options: { videoId: number }) => void | 30 | 'live-ready': (options: { videoUUID: string }) => void |
32 | 31 | ||
33 | 'bad-socket-health': (options: { videoId: number }) => void | 32 | 'bad-socket-health': (options: { videoUUID: string }) => void |
34 | 'duration-exceeded': (options: { videoId: number }) => void | 33 | 'duration-exceeded': (options: { videoUUID: string }) => void |
35 | 'quota-exceeded': (options: { videoId: number }) => void | 34 | 'quota-exceeded': (options: { videoUUID: string }) => void |
36 | 35 | ||
37 | 'ffmpeg-end': (options: { videoId: number }) => void | 36 | 'transcoding-end': (options: { videoUUID: string }) => void |
38 | 'ffmpeg-error': (options: { videoId: number }) => void | 37 | 'transcoding-error': (options: { videoUUID: string }) => void |
39 | 38 | ||
40 | 'after-cleanup': (options: { videoId: number }) => void | 39 | 'after-cleanup': (options: { videoUUID: string }) => void |
41 | } | 40 | } |
42 | 41 | ||
43 | declare interface MuxingSession { | 42 | declare interface MuxingSession { |
@@ -52,7 +51,7 @@ declare interface MuxingSession { | |||
52 | 51 | ||
53 | class MuxingSession extends EventEmitter { | 52 | class MuxingSession extends EventEmitter { |
54 | 53 | ||
55 | private ffmpegCommand: FfmpegCommand | 54 | private transcodingWrapper: AbstractTranscodingWrapper |
56 | 55 | ||
57 | private readonly context: any | 56 | private readonly context: any |
58 | private readonly user: MUserId | 57 | private readonly user: MUserId |
@@ -67,7 +66,6 @@ class MuxingSession extends EventEmitter { | |||
67 | 66 | ||
68 | private readonly hasAudio: boolean | 67 | private readonly hasAudio: boolean |
69 | 68 | ||
70 | private readonly videoId: number | ||
71 | private readonly videoUUID: string | 69 | private readonly videoUUID: string |
72 | private readonly saveReplay: boolean | 70 | private readonly saveReplay: boolean |
73 | 71 | ||
@@ -126,7 +124,6 @@ class MuxingSession extends EventEmitter { | |||
126 | 124 | ||
127 | this.allResolutions = options.allResolutions | 125 | this.allResolutions = options.allResolutions |
128 | 126 | ||
129 | this.videoId = this.videoLive.Video.id | ||
130 | this.videoUUID = this.videoLive.Video.uuid | 127 | this.videoUUID = this.videoLive.Video.uuid |
131 | 128 | ||
132 | this.saveReplay = this.videoLive.saveReplay | 129 | this.saveReplay = this.videoLive.saveReplay |
@@ -145,63 +142,23 @@ class MuxingSession extends EventEmitter { | |||
145 | 142 | ||
146 | await this.prepareDirectories() | 143 | await this.prepareDirectories() |
147 | 144 | ||
148 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | 145 | this.transcodingWrapper = this.buildTranscodingWrapper() |
149 | ? await getLiveTranscodingCommand({ | ||
150 | inputUrl: this.inputUrl, | ||
151 | 146 | ||
152 | outPath: this.outDirectory, | 147 | this.transcodingWrapper.on('end', () => this.onTranscodedEnded()) |
153 | masterPlaylistName: this.streamingPlaylist.playlistFilename, | 148 | this.transcodingWrapper.on('error', () => this.onTranscodingError()) |
154 | 149 | ||
155 | latencyMode: this.videoLive.latencyMode, | 150 | await this.transcodingWrapper.run() |
156 | |||
157 | resolutions: this.allResolutions, | ||
158 | fps: this.fps, | ||
159 | bitrate: this.bitrate, | ||
160 | ratio: this.ratio, | ||
161 | |||
162 | hasAudio: this.hasAudio, | ||
163 | |||
164 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
165 | profile: CONFIG.LIVE.TRANSCODING.PROFILE | ||
166 | }) | ||
167 | : getLiveMuxingCommand({ | ||
168 | inputUrl: this.inputUrl, | ||
169 | outPath: this.outDirectory, | ||
170 | masterPlaylistName: this.streamingPlaylist.playlistFilename, | ||
171 | latencyMode: this.videoLive.latencyMode | ||
172 | }) | ||
173 | |||
174 | logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags()) | ||
175 | 151 | ||
176 | this.watchMasterFile() | 152 | this.watchMasterFile() |
177 | this.watchTSFiles() | 153 | this.watchTSFiles() |
178 | this.watchM3U8File() | 154 | this.watchM3U8File() |
179 | |||
180 | let ffmpegShellCommand: string | ||
181 | this.ffmpegCommand.on('start', cmdline => { | ||
182 | ffmpegShellCommand = cmdline | ||
183 | |||
184 | logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) | ||
185 | }) | ||
186 | |||
187 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { | ||
188 | this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) | ||
189 | }) | ||
190 | |||
191 | this.ffmpegCommand.on('end', () => { | ||
192 | this.emit('ffmpeg-end', ({ videoId: this.videoId })) | ||
193 | |||
194 | this.onFFmpegEnded() | ||
195 | }) | ||
196 | |||
197 | this.ffmpegCommand.run() | ||
198 | } | 155 | } |
199 | 156 | ||
200 | abort () { | 157 | abort () { |
201 | if (!this.ffmpegCommand) return | 158 | if (!this.transcodingWrapper) return |
202 | 159 | ||
203 | this.aborted = true | 160 | this.aborted = true |
204 | this.ffmpegCommand.kill('SIGINT') | 161 | this.transcodingWrapper.abort() |
205 | } | 162 | } |
206 | 163 | ||
207 | destroy () { | 164 | destroy () { |
@@ -210,48 +167,6 @@ class MuxingSession extends EventEmitter { | |||
210 | this.hasClientSocketInBadHealthWithCache.clear() | 167 | this.hasClientSocketInBadHealthWithCache.clear() |
211 | } | 168 | } |
212 | 169 | ||
213 | private onFFmpegError (options: { | ||
214 | err: any | ||
215 | stdout: string | ||
216 | stderr: string | ||
217 | ffmpegShellCommand: string | ||
218 | }) { | ||
219 | const { err, stdout, stderr, ffmpegShellCommand } = options | ||
220 | |||
221 | this.onFFmpegEnded() | ||
222 | |||
223 | // Don't care that we killed the ffmpeg process | ||
224 | if (err?.message?.includes('Exiting normally')) return | ||
225 | |||
226 | logger.error('Live transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) | ||
227 | |||
228 | this.emit('ffmpeg-error', ({ videoId: this.videoId })) | ||
229 | } | ||
230 | |||
231 | private onFFmpegEnded () { | ||
232 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags()) | ||
233 | |||
234 | setTimeout(() => { | ||
235 | // Wait latest segments generation, and close watchers | ||
236 | |||
237 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) | ||
238 | .then(() => { | ||
239 | // Process remaining segments hash | ||
240 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | ||
241 | this.processSegments(this.segmentsToProcessPerPlaylist[key]) | ||
242 | } | ||
243 | }) | ||
244 | .catch(err => { | ||
245 | logger.error( | ||
246 | 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, | ||
247 | { err, ...this.lTags() } | ||
248 | ) | ||
249 | }) | ||
250 | |||
251 | this.emit('after-cleanup', { videoId: this.videoId }) | ||
252 | }, 1000) | ||
253 | } | ||
254 | |||
255 | private watchMasterFile () { | 170 | private watchMasterFile () { |
256 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) | 171 | this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename) |
257 | 172 | ||
@@ -272,6 +187,8 @@ class MuxingSession extends EventEmitter { | |||
272 | 187 | ||
273 | this.masterPlaylistCreated = true | 188 | this.masterPlaylistCreated = true |
274 | 189 | ||
190 | logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags()) | ||
191 | |||
275 | this.masterWatcher.close() | 192 | this.masterWatcher.close() |
276 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) | 193 | .catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() })) |
277 | }) | 194 | }) |
@@ -318,19 +235,19 @@ class MuxingSession extends EventEmitter { | |||
318 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] | 235 | this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] |
319 | 236 | ||
320 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { | 237 | if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { |
321 | this.emit('bad-socket-health', { videoId: this.videoId }) | 238 | this.emit('bad-socket-health', { videoUUID: this.videoUUID }) |
322 | return | 239 | return |
323 | } | 240 | } |
324 | 241 | ||
325 | // Duration constraint check | 242 | // Duration constraint check |
326 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { | 243 | if (this.isDurationConstraintValid(startStreamDateTime) !== true) { |
327 | this.emit('duration-exceeded', { videoId: this.videoId }) | 244 | this.emit('duration-exceeded', { videoUUID: this.videoUUID }) |
328 | return | 245 | return |
329 | } | 246 | } |
330 | 247 | ||
331 | // Check user quota if the user enabled replay saving | 248 | // Check user quota if the user enabled replay saving |
332 | if (await this.isQuotaExceeded(segmentPath) === true) { | 249 | if (await this.isQuotaExceeded(segmentPath) === true) { |
333 | this.emit('quota-exceeded', { videoId: this.videoId }) | 250 | this.emit('quota-exceeded', { videoUUID: this.videoUUID }) |
334 | } | 251 | } |
335 | } | 252 | } |
336 | 253 | ||
@@ -438,10 +355,40 @@ class MuxingSession extends EventEmitter { | |||
438 | if (this.masterPlaylistCreated && !this.liveReady) { | 355 | if (this.masterPlaylistCreated && !this.liveReady) { |
439 | this.liveReady = true | 356 | this.liveReady = true |
440 | 357 | ||
441 | this.emit('live-ready', { videoId: this.videoId }) | 358 | this.emit('live-ready', { videoUUID: this.videoUUID }) |
442 | } | 359 | } |
443 | } | 360 | } |
444 | 361 | ||
362 | private onTranscodingError () { | ||
363 | this.emit('transcoding-error', ({ videoUUID: this.videoUUID })) | ||
364 | } | ||
365 | |||
366 | private onTranscodedEnded () { | ||
367 | this.emit('transcoding-end', ({ videoUUID: this.videoUUID })) | ||
368 | |||
369 | logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags()) | ||
370 | |||
371 | setTimeout(() => { | ||
372 | // Wait latest segments generation, and close watchers | ||
373 | |||
374 | Promise.all([ this.tsWatcher.close(), this.masterWatcher.close(), this.m3u8Watcher.close() ]) | ||
375 | .then(() => { | ||
376 | // Process remaining segments hash | ||
377 | for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { | ||
378 | this.processSegments(this.segmentsToProcessPerPlaylist[key]) | ||
379 | } | ||
380 | }) | ||
381 | .catch(err => { | ||
382 | logger.error( | ||
383 | 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, | ||
384 | { err, ...this.lTags() } | ||
385 | ) | ||
386 | }) | ||
387 | |||
388 | this.emit('after-cleanup', { videoUUID: this.videoUUID }) | ||
389 | }, 1000) | ||
390 | } | ||
391 | |||
445 | private hasClientSocketInBadHealth (sessionId: string) { | 392 | private hasClientSocketInBadHealth (sessionId: string) { |
446 | const rtmpSession = this.context.sessions.get(sessionId) | 393 | const rtmpSession = this.context.sessions.get(sessionId) |
447 | 394 | ||
@@ -503,6 +450,36 @@ class MuxingSession extends EventEmitter { | |||
503 | sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED | 450 | sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED |
504 | }) | 451 | }) |
505 | } | 452 | } |
453 | |||
454 | private buildTranscodingWrapper () { | ||
455 | const options = { | ||
456 | streamingPlaylist: this.streamingPlaylist, | ||
457 | videoLive: this.videoLive, | ||
458 | |||
459 | lTags: this.lTags, | ||
460 | |||
461 | inputUrl: this.inputUrl, | ||
462 | |||
463 | toTranscode: this.allResolutions.map(resolution => ({ | ||
464 | resolution, | ||
465 | fps: computeOutputFPS({ inputFPS: this.fps, resolution }) | ||
466 | })), | ||
467 | |||
468 | fps: this.fps, | ||
469 | bitrate: this.bitrate, | ||
470 | ratio: this.ratio, | ||
471 | hasAudio: this.hasAudio, | ||
472 | |||
473 | segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, | ||
474 | segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode), | ||
475 | |||
476 | outDirectory: this.outDirectory | ||
477 | } | ||
478 | |||
479 | return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
480 | ? new RemoteTranscodingWrapper(options) | ||
481 | : new FFmpegTranscodingWrapper(options) | ||
482 | } | ||
506 | } | 483 | } |
507 | 484 | ||
508 | // --------------------------------------------------------------------------- | 485 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts new file mode 100644 index 000000000..226ba4573 --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts | |||
@@ -0,0 +1,101 @@ | |||
1 | import EventEmitter from 'events' | ||
2 | import { LoggerTagsFn } from '@server/helpers/logger' | ||
3 | import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models' | ||
4 | import { LiveVideoError } from '@shared/models' | ||
5 | |||
6 | interface TranscodingWrapperEvents { | ||
7 | 'end': () => void | ||
8 | |||
9 | 'error': (options: { err: Error }) => void | ||
10 | } | ||
11 | |||
12 | declare interface AbstractTranscodingWrapper { | ||
13 | on<U extends keyof TranscodingWrapperEvents>( | ||
14 | event: U, listener: TranscodingWrapperEvents[U] | ||
15 | ): this | ||
16 | |||
17 | emit<U extends keyof TranscodingWrapperEvents>( | ||
18 | event: U, ...args: Parameters<TranscodingWrapperEvents[U]> | ||
19 | ): boolean | ||
20 | } | ||
21 | |||
22 | interface AbstractTranscodingWrapperOptions { | ||
23 | streamingPlaylist: MStreamingPlaylistVideo | ||
24 | videoLive: MVideoLiveVideo | ||
25 | |||
26 | lTags: LoggerTagsFn | ||
27 | |||
28 | inputUrl: string | ||
29 | fps: number | ||
30 | toTranscode: { | ||
31 | resolution: number | ||
32 | fps: number | ||
33 | }[] | ||
34 | |||
35 | bitrate: number | ||
36 | ratio: number | ||
37 | hasAudio: boolean | ||
38 | |||
39 | segmentListSize: number | ||
40 | segmentDuration: number | ||
41 | |||
42 | outDirectory: string | ||
43 | } | ||
44 | |||
45 | abstract class AbstractTranscodingWrapper extends EventEmitter { | ||
46 | protected readonly videoLive: MVideoLiveVideo | ||
47 | |||
48 | protected readonly toTranscode: { | ||
49 | resolution: number | ||
50 | fps: number | ||
51 | }[] | ||
52 | |||
53 | protected readonly inputUrl: string | ||
54 | protected readonly fps: number | ||
55 | protected readonly bitrate: number | ||
56 | protected readonly ratio: number | ||
57 | protected readonly hasAudio: boolean | ||
58 | |||
59 | protected readonly segmentListSize: number | ||
60 | protected readonly segmentDuration: number | ||
61 | |||
62 | protected readonly videoUUID: string | ||
63 | |||
64 | protected readonly outDirectory: string | ||
65 | |||
66 | protected readonly lTags: LoggerTagsFn | ||
67 | |||
68 | protected readonly streamingPlaylist: MStreamingPlaylistVideo | ||
69 | |||
70 | constructor (options: AbstractTranscodingWrapperOptions) { | ||
71 | super() | ||
72 | |||
73 | this.lTags = options.lTags | ||
74 | |||
75 | this.videoLive = options.videoLive | ||
76 | this.videoUUID = options.videoLive.Video.uuid | ||
77 | this.streamingPlaylist = options.streamingPlaylist | ||
78 | |||
79 | this.inputUrl = options.inputUrl | ||
80 | this.fps = options.fps | ||
81 | this.toTranscode = options.toTranscode | ||
82 | |||
83 | this.bitrate = options.bitrate | ||
84 | this.ratio = options.ratio | ||
85 | this.hasAudio = options.hasAudio | ||
86 | |||
87 | this.segmentListSize = options.segmentListSize | ||
88 | this.segmentDuration = options.segmentDuration | ||
89 | |||
90 | this.outDirectory = options.outDirectory | ||
91 | } | ||
92 | |||
93 | abstract run (): Promise<void> | ||
94 | |||
95 | abstract abort (error?: LiveVideoError): void | ||
96 | } | ||
97 | |||
98 | export { | ||
99 | AbstractTranscodingWrapper, | ||
100 | AbstractTranscodingWrapperOptions | ||
101 | } | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts new file mode 100644 index 000000000..1f4c12bd4 --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts | |||
@@ -0,0 +1,95 @@ | |||
1 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
2 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { VIDEO_LIVE } from '@server/initializers/constants' | ||
6 | import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' | ||
7 | import { FFmpegLive } from '@shared/ffmpeg' | ||
8 | import { getLiveSegmentTime } from '../../live-utils' | ||
9 | import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' | ||
10 | |||
11 | export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper { | ||
12 | private ffmpegCommand: FfmpegCommand | ||
13 | private ended = false | ||
14 | |||
15 | async run () { | ||
16 | this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED | ||
17 | ? await this.buildFFmpegLive().getLiveTranscodingCommand({ | ||
18 | inputUrl: this.inputUrl, | ||
19 | |||
20 | outPath: this.outDirectory, | ||
21 | masterPlaylistName: this.streamingPlaylist.playlistFilename, | ||
22 | |||
23 | segmentListSize: this.segmentListSize, | ||
24 | segmentDuration: this.segmentDuration, | ||
25 | |||
26 | toTranscode: this.toTranscode, | ||
27 | |||
28 | bitrate: this.bitrate, | ||
29 | ratio: this.ratio, | ||
30 | |||
31 | hasAudio: this.hasAudio | ||
32 | }) | ||
33 | : this.buildFFmpegLive().getLiveMuxingCommand({ | ||
34 | inputUrl: this.inputUrl, | ||
35 | outPath: this.outDirectory, | ||
36 | |||
37 | masterPlaylistName: this.streamingPlaylist.playlistFilename, | ||
38 | |||
39 | segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, | ||
40 | segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode) | ||
41 | }) | ||
42 | |||
43 | logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags()) | ||
44 | |||
45 | this.ffmpegCommand.run() | ||
46 | |||
47 | let ffmpegShellCommand: string | ||
48 | this.ffmpegCommand.on('start', cmdline => { | ||
49 | ffmpegShellCommand = cmdline | ||
50 | |||
51 | logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) | ||
52 | }) | ||
53 | |||
54 | this.ffmpegCommand.on('error', (err, stdout, stderr) => { | ||
55 | this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) | ||
56 | }) | ||
57 | |||
58 | this.ffmpegCommand.on('end', () => { | ||
59 | this.onFFmpegEnded() | ||
60 | }) | ||
61 | |||
62 | this.ffmpegCommand.run() | ||
63 | } | ||
64 | |||
65 | abort () { | ||
66 | // Nothing to do, ffmpeg will automatically exit | ||
67 | } | ||
68 | |||
69 | private onFFmpegError (options: { | ||
70 | err: any | ||
71 | stdout: string | ||
72 | stderr: string | ||
73 | ffmpegShellCommand: string | ||
74 | }) { | ||
75 | const { err, stdout, stderr, ffmpegShellCommand } = options | ||
76 | |||
77 | // Don't care that we killed the ffmpeg process | ||
78 | if (err?.message?.includes('Exiting normally')) return | ||
79 | |||
80 | logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) | ||
81 | |||
82 | this.emit('error', { err }) | ||
83 | } | ||
84 | |||
85 | private onFFmpegEnded () { | ||
86 | if (this.ended) return | ||
87 | |||
88 | this.ended = true | ||
89 | this.emit('end') | ||
90 | } | ||
91 | |||
92 | private buildFFmpegLive () { | ||
93 | return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) | ||
94 | } | ||
95 | } | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/index.ts b/server/lib/live/shared/transcoding-wrapper/index.ts new file mode 100644 index 000000000..ae28fa1ca --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './abstract-transcoding-wrapper' | ||
2 | export * from './ffmpeg-transcoding-wrapper' | ||
3 | export * from './remote-transcoding-wrapper' | ||
diff --git a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts new file mode 100644 index 000000000..345eaf442 --- /dev/null +++ b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners' | ||
2 | import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' | ||
3 | |||
4 | export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper { | ||
5 | async run () { | ||
6 | await new LiveRTMPHLSTranscodingJobHandler().create({ | ||
7 | rtmpUrl: this.inputUrl, | ||
8 | toTranscode: this.toTranscode, | ||
9 | video: this.videoLive.Video, | ||
10 | outputDirectory: this.outDirectory, | ||
11 | playlist: this.streamingPlaylist, | ||
12 | segmentListSize: this.segmentListSize, | ||
13 | segmentDuration: this.segmentDuration | ||
14 | }) | ||
15 | } | ||
16 | |||
17 | abort () { | ||
18 | this.emit('end') | ||
19 | } | ||
20 | } | ||
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts index 8b413a40e..6525f8dfb 100644 --- a/server/lib/object-storage/index.ts +++ b/server/lib/object-storage/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | export * from './keys' | 1 | export * from './keys' |
2 | export * from './proxy' | ||
2 | export * from './urls' | 3 | export * from './urls' |
3 | export * from './videos' | 4 | export * from './videos' |
diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts new file mode 100644 index 000000000..c782a8a25 --- /dev/null +++ b/server/lib/object-storage/proxy.ts | |||
@@ -0,0 +1,97 @@ | |||
1 | import express from 'express' | ||
2 | import { PassThrough, pipeline } from 'stream' | ||
3 | import { GetObjectCommandOutput } from '@aws-sdk/client-s3' | ||
4 | import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist' | ||
5 | import { logger } from '@server/helpers/logger' | ||
6 | import { StreamReplacer } from '@server/helpers/stream-replacer' | ||
7 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
8 | import { HttpStatusCode } from '@shared/models' | ||
9 | import { injectQueryToPlaylistUrls } from '../hls' | ||
10 | import { getHLSFileReadStream, getWebTorrentFileReadStream } from './videos' | ||
11 | |||
12 | export async function proxifyWebTorrentFile (options: { | ||
13 | req: express.Request | ||
14 | res: express.Response | ||
15 | filename: string | ||
16 | }) { | ||
17 | const { req, res, filename } = options | ||
18 | |||
19 | logger.debug('Proxifying WebTorrent file %s from object storage.', filename) | ||
20 | |||
21 | try { | ||
22 | const { response: s3Response, stream } = await getWebTorrentFileReadStream({ | ||
23 | filename, | ||
24 | rangeHeader: req.header('range') | ||
25 | }) | ||
26 | |||
27 | setS3Headers(res, s3Response) | ||
28 | |||
29 | return stream.pipe(res) | ||
30 | } catch (err) { | ||
31 | return handleObjectStorageFailure(res, err) | ||
32 | } | ||
33 | } | ||
34 | |||
35 | export async function proxifyHLS (options: { | ||
36 | req: express.Request | ||
37 | res: express.Response | ||
38 | playlist: MStreamingPlaylist | ||
39 | video: MVideo | ||
40 | filename: string | ||
41 | reinjectVideoFileToken: boolean | ||
42 | }) { | ||
43 | const { req, res, playlist, video, filename, reinjectVideoFileToken } = options | ||
44 | |||
45 | logger.debug('Proxifying HLS file %s from object storage.', filename) | ||
46 | |||
47 | try { | ||
48 | const { response: s3Response, stream } = await getHLSFileReadStream({ | ||
49 | playlist: playlist.withVideo(video), | ||
50 | filename, | ||
51 | rangeHeader: req.header('range') | ||
52 | }) | ||
53 | |||
54 | setS3Headers(res, s3Response) | ||
55 | |||
56 | const streamReplacer = reinjectVideoFileToken | ||
57 | ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) | ||
58 | : new PassThrough() | ||
59 | |||
60 | return pipeline( | ||
61 | stream, | ||
62 | streamReplacer, | ||
63 | res, | ||
64 | err => { | ||
65 | if (!err) return | ||
66 | |||
67 | handleObjectStorageFailure(res, err) | ||
68 | } | ||
69 | ) | ||
70 | } catch (err) { | ||
71 | return handleObjectStorageFailure(res, err) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | // --------------------------------------------------------------------------- | ||
76 | // Private | ||
77 | // --------------------------------------------------------------------------- | ||
78 | |||
79 | function handleObjectStorageFailure (res: express.Response, err: Error) { | ||
80 | if (err.name === 'NoSuchKey') { | ||
81 | logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) | ||
82 | return res.sendStatus(HttpStatusCode.NOT_FOUND_404) | ||
83 | } | ||
84 | |||
85 | return res.fail({ | ||
86 | status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, | ||
87 | message: err.message, | ||
88 | type: err.name | ||
89 | }) | ||
90 | } | ||
91 | |||
92 | function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { | ||
93 | if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { | ||
94 | res.setHeader('Content-Range', s3Response.ContentRange) | ||
95 | res.status(HttpStatusCode.PARTIAL_CONTENT_206) | ||
96 | } | ||
97 | } | ||
diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts index 0398ca61d..ded7e9743 100644 --- a/server/lib/peertube-socket.ts +++ b/server/lib/peertube-socket.ts | |||
@@ -2,10 +2,12 @@ import { Server as HTTPServer } from 'http' | |||
2 | import { Namespace, Server as SocketServer, Socket } from 'socket.io' | 2 | import { Namespace, Server as SocketServer, Socket } from 'socket.io' |
3 | import { isIdValid } from '@server/helpers/custom-validators/misc' | 3 | import { isIdValid } from '@server/helpers/custom-validators/misc' |
4 | import { MVideo, MVideoImmutable } from '@server/types/models' | 4 | import { MVideo, MVideoImmutable } from '@server/types/models' |
5 | import { MRunner } from '@server/types/models/runners' | ||
5 | import { UserNotificationModelForApi } from '@server/types/models/user' | 6 | import { UserNotificationModelForApi } from '@server/types/models/user' |
6 | import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models' | 7 | import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models' |
7 | import { logger } from '../helpers/logger' | 8 | import { logger } from '../helpers/logger' |
8 | import { authenticateSocket } from '../middlewares' | 9 | import { authenticateRunnerSocket, authenticateSocket } from '../middlewares' |
10 | import { Debounce } from '@server/helpers/debounce' | ||
9 | 11 | ||
10 | class PeerTubeSocket { | 12 | class PeerTubeSocket { |
11 | 13 | ||
@@ -13,6 +15,7 @@ class PeerTubeSocket { | |||
13 | 15 | ||
14 | private userNotificationSockets: { [ userId: number ]: Socket[] } = {} | 16 | private userNotificationSockets: { [ userId: number ]: Socket[] } = {} |
15 | private liveVideosNamespace: Namespace | 17 | private liveVideosNamespace: Namespace |
18 | private readonly runnerSockets = new Set<Socket>() | ||
16 | 19 | ||
17 | private constructor () {} | 20 | private constructor () {} |
18 | 21 | ||
@@ -24,7 +27,7 @@ class PeerTubeSocket { | |||
24 | .on('connection', socket => { | 27 | .on('connection', socket => { |
25 | const userId = socket.handshake.auth.user.id | 28 | const userId = socket.handshake.auth.user.id |
26 | 29 | ||
27 | logger.debug('User %d connected on the notification system.', userId) | 30 | logger.debug('User %d connected to the notification system.', userId) |
28 | 31 | ||
29 | if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] | 32 | if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] |
30 | 33 | ||
@@ -53,6 +56,22 @@ class PeerTubeSocket { | |||
53 | socket.leave(videoId) | 56 | socket.leave(videoId) |
54 | }) | 57 | }) |
55 | }) | 58 | }) |
59 | |||
60 | io.of('/runners') | ||
61 | .use(authenticateRunnerSocket) | ||
62 | .on('connection', socket => { | ||
63 | const runner: MRunner = socket.handshake.auth.runner | ||
64 | |||
65 | logger.debug(`New runner "${runner.name}" connected to the notification system.`) | ||
66 | |||
67 | this.runnerSockets.add(socket) | ||
68 | |||
69 | socket.on('disconnect', () => { | ||
70 | logger.debug(`Runner "${runner.name}" disconnected from the notification system.`) | ||
71 | |||
72 | this.runnerSockets.delete(socket) | ||
73 | }) | ||
74 | }) | ||
56 | } | 75 | } |
57 | 76 | ||
58 | sendNotification (userId: number, notification: UserNotificationModelForApi) { | 77 | sendNotification (userId: number, notification: UserNotificationModelForApi) { |
@@ -89,6 +108,15 @@ class PeerTubeSocket { | |||
89 | .emit(type, data) | 108 | .emit(type, data) |
90 | } | 109 | } |
91 | 110 | ||
111 | @Debounce({ timeoutMS: 1000 }) | ||
112 | sendAvailableJobsPingToRunners () { | ||
113 | logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`) | ||
114 | |||
115 | for (const runners of this.runnerSockets) { | ||
116 | runners.emit('available-jobs') | ||
117 | } | ||
118 | } | ||
119 | |||
92 | static get Instance () { | 120 | static get Instance () { |
93 | return this.instance || (this.instance = new this()) | 121 | return this.instance || (this.instance = new this()) |
94 | } | 122 | } |
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts index 66383af46..92ef87cca 100644 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ b/server/lib/plugins/plugin-helpers-builder.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { Server } from 'http' | 2 | import { Server } from 'http' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { ffprobePromise } from '@server/helpers/ffmpeg/ffprobe-utils' | ||
5 | import { buildLogger } from '@server/helpers/logger' | 4 | import { buildLogger } from '@server/helpers/logger' |
6 | import { CONFIG } from '@server/initializers/config' | 5 | import { CONFIG } from '@server/initializers/config' |
7 | import { WEBSERVER } from '@server/initializers/constants' | 6 | import { WEBSERVER } from '@server/initializers/constants' |
@@ -16,6 +15,7 @@ import { VideoModel } from '@server/models/video/video' | |||
16 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' | 15 | import { VideoBlacklistModel } from '@server/models/video/video-blacklist' |
17 | import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' | 16 | import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' |
18 | import { PeerTubeHelpers } from '@server/types/plugins' | 17 | import { PeerTubeHelpers } from '@server/types/plugins' |
18 | import { ffprobePromise } from '@shared/ffmpeg' | ||
19 | import { VideoBlacklistCreate, VideoStorage } from '@shared/models' | 19 | import { VideoBlacklistCreate, VideoStorage } from '@shared/models' |
20 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' | 20 | import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' |
21 | import { PeerTubeSocket } from '../peertube-socket' | 21 | import { PeerTubeSocket } from '../peertube-socket' |
diff --git a/server/lib/runners/index.ts b/server/lib/runners/index.ts new file mode 100644 index 000000000..a737c7b59 --- /dev/null +++ b/server/lib/runners/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './job-handlers' | ||
2 | export * from './runner' | ||
3 | export * from './runner-urls' | ||
diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts new file mode 100644 index 000000000..73fc14574 --- /dev/null +++ b/server/lib/runners/job-handlers/abstract-job-handler.ts | |||
@@ -0,0 +1,271 @@ | |||
1 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { RUNNER_JOBS } from '@server/initializers/constants' | ||
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
6 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
7 | import { setAsUpdated } from '@server/models/shared' | ||
8 | import { MRunnerJob } from '@server/types/models/runners' | ||
9 | import { pick } from '@shared/core-utils' | ||
10 | import { | ||
11 | RunnerJobLiveRTMPHLSTranscodingPayload, | ||
12 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload, | ||
13 | RunnerJobState, | ||
14 | RunnerJobSuccessPayload, | ||
15 | RunnerJobType, | ||
16 | RunnerJobUpdatePayload, | ||
17 | RunnerJobVODAudioMergeTranscodingPayload, | ||
18 | RunnerJobVODAudioMergeTranscodingPrivatePayload, | ||
19 | RunnerJobVODHLSTranscodingPayload, | ||
20 | RunnerJobVODHLSTranscodingPrivatePayload, | ||
21 | RunnerJobVODWebVideoTranscodingPayload, | ||
22 | RunnerJobVODWebVideoTranscodingPrivatePayload | ||
23 | } from '@shared/models' | ||
24 | |||
25 | type CreateRunnerJobArg = | ||
26 | { | ||
27 | type: Extract<RunnerJobType, 'vod-web-video-transcoding'> | ||
28 | payload: RunnerJobVODWebVideoTranscodingPayload | ||
29 | privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | ||
30 | } | | ||
31 | { | ||
32 | type: Extract<RunnerJobType, 'vod-hls-transcoding'> | ||
33 | payload: RunnerJobVODHLSTranscodingPayload | ||
34 | privatePayload: RunnerJobVODHLSTranscodingPrivatePayload | ||
35 | } | | ||
36 | { | ||
37 | type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'> | ||
38 | payload: RunnerJobVODAudioMergeTranscodingPayload | ||
39 | privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload | ||
40 | } | | ||
41 | { | ||
42 | type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'> | ||
43 | payload: RunnerJobLiveRTMPHLSTranscodingPayload | ||
44 | privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
45 | } | ||
46 | |||
47 | export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> { | ||
48 | |||
49 | protected readonly lTags = loggerTagsFactory('runner') | ||
50 | |||
51 | // --------------------------------------------------------------------------- | ||
52 | |||
53 | abstract create (options: C): Promise<MRunnerJob> | ||
54 | |||
55 | protected async createRunnerJob (options: CreateRunnerJobArg & { | ||
56 | jobUUID: string | ||
57 | priority: number | ||
58 | dependsOnRunnerJob?: MRunnerJob | ||
59 | }): Promise<MRunnerJob> { | ||
60 | const { priority, dependsOnRunnerJob } = options | ||
61 | |||
62 | const runnerJob = new RunnerJobModel({ | ||
63 | ...pick(options, [ 'type', 'payload', 'privatePayload' ]), | ||
64 | |||
65 | uuid: options.jobUUID, | ||
66 | |||
67 | state: dependsOnRunnerJob | ||
68 | ? RunnerJobState.WAITING_FOR_PARENT_JOB | ||
69 | : RunnerJobState.PENDING, | ||
70 | |||
71 | dependsOnRunnerJobId: dependsOnRunnerJob?.id, | ||
72 | |||
73 | priority | ||
74 | }) | ||
75 | |||
76 | const job = await sequelizeTypescript.transaction(async transaction => { | ||
77 | return runnerJob.save({ transaction }) | ||
78 | }) | ||
79 | |||
80 | if (runnerJob.state === RunnerJobState.PENDING) { | ||
81 | PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() | ||
82 | } | ||
83 | |||
84 | return job | ||
85 | } | ||
86 | |||
87 | // --------------------------------------------------------------------------- | ||
88 | |||
89 | protected abstract specificUpdate (options: { | ||
90 | runnerJob: MRunnerJob | ||
91 | updatePayload?: U | ||
92 | }): Promise<void> | void | ||
93 | |||
94 | async update (options: { | ||
95 | runnerJob: MRunnerJob | ||
96 | progress?: number | ||
97 | updatePayload?: U | ||
98 | }) { | ||
99 | const { runnerJob, progress } = options | ||
100 | |||
101 | await this.specificUpdate(options) | ||
102 | |||
103 | if (progress) runnerJob.progress = progress | ||
104 | |||
105 | await retryTransactionWrapper(() => { | ||
106 | return sequelizeTypescript.transaction(async transaction => { | ||
107 | if (runnerJob.changed()) { | ||
108 | return runnerJob.save({ transaction }) | ||
109 | } | ||
110 | |||
111 | // Don't update the job too often | ||
112 | if (new Date().getTime() - runnerJob.updatedAt.getTime() > 2000) { | ||
113 | await setAsUpdated({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id, transaction }) | ||
114 | } | ||
115 | }) | ||
116 | }) | ||
117 | } | ||
118 | |||
119 | // --------------------------------------------------------------------------- | ||
120 | |||
121 | async complete (options: { | ||
122 | runnerJob: MRunnerJob | ||
123 | resultPayload: S | ||
124 | }) { | ||
125 | const { runnerJob } = options | ||
126 | |||
127 | try { | ||
128 | await this.specificComplete(options) | ||
129 | |||
130 | runnerJob.state = RunnerJobState.COMPLETED | ||
131 | } catch (err) { | ||
132 | logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) | ||
133 | |||
134 | runnerJob.state = RunnerJobState.ERRORED | ||
135 | runnerJob.error = err.message | ||
136 | } | ||
137 | |||
138 | runnerJob.progress = null | ||
139 | runnerJob.finishedAt = new Date() | ||
140 | |||
141 | await retryTransactionWrapper(() => { | ||
142 | return sequelizeTypescript.transaction(async transaction => { | ||
143 | await runnerJob.save({ transaction }) | ||
144 | }) | ||
145 | }) | ||
146 | |||
147 | const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob) | ||
148 | |||
149 | if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() | ||
150 | } | ||
151 | |||
152 | protected abstract specificComplete (options: { | ||
153 | runnerJob: MRunnerJob | ||
154 | resultPayload: S | ||
155 | }): Promise<void> | void | ||
156 | |||
157 | // --------------------------------------------------------------------------- | ||
158 | |||
159 | async cancel (options: { | ||
160 | runnerJob: MRunnerJob | ||
161 | fromParent?: boolean | ||
162 | }) { | ||
163 | const { runnerJob, fromParent } = options | ||
164 | |||
165 | await this.specificCancel(options) | ||
166 | |||
167 | const cancelState = fromParent | ||
168 | ? RunnerJobState.PARENT_CANCELLED | ||
169 | : RunnerJobState.CANCELLED | ||
170 | |||
171 | runnerJob.setToErrorOrCancel(cancelState) | ||
172 | |||
173 | await retryTransactionWrapper(() => { | ||
174 | return sequelizeTypescript.transaction(async transaction => { | ||
175 | await runnerJob.save({ transaction }) | ||
176 | }) | ||
177 | }) | ||
178 | |||
179 | const children = await RunnerJobModel.listChildrenOf(runnerJob) | ||
180 | for (const child of children) { | ||
181 | logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid)) | ||
182 | |||
183 | await this.cancel({ runnerJob: child, fromParent: true }) | ||
184 | } | ||
185 | } | ||
186 | |||
187 | protected abstract specificCancel (options: { | ||
188 | runnerJob: MRunnerJob | ||
189 | }): Promise<void> | void | ||
190 | |||
191 | // --------------------------------------------------------------------------- | ||
192 | |||
193 | protected abstract isAbortSupported (): boolean | ||
194 | |||
195 | async abort (options: { | ||
196 | runnerJob: MRunnerJob | ||
197 | }) { | ||
198 | const { runnerJob } = options | ||
199 | |||
200 | if (this.isAbortSupported() !== true) { | ||
201 | return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' }) | ||
202 | } | ||
203 | |||
204 | await this.specificAbort(options) | ||
205 | |||
206 | runnerJob.resetToPending() | ||
207 | |||
208 | await retryTransactionWrapper(() => { | ||
209 | return sequelizeTypescript.transaction(async transaction => { | ||
210 | await runnerJob.save({ transaction }) | ||
211 | }) | ||
212 | }) | ||
213 | } | ||
214 | |||
215 | protected setAbortState (runnerJob: MRunnerJob) { | ||
216 | runnerJob.resetToPending() | ||
217 | } | ||
218 | |||
219 | protected abstract specificAbort (options: { | ||
220 | runnerJob: MRunnerJob | ||
221 | }): Promise<void> | void | ||
222 | |||
223 | // --------------------------------------------------------------------------- | ||
224 | |||
225 | async error (options: { | ||
226 | runnerJob: MRunnerJob | ||
227 | message: string | ||
228 | fromParent?: boolean | ||
229 | }) { | ||
230 | const { runnerJob, message, fromParent } = options | ||
231 | |||
232 | const errorState = fromParent | ||
233 | ? RunnerJobState.PARENT_ERRORED | ||
234 | : RunnerJobState.ERRORED | ||
235 | |||
236 | const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES | ||
237 | ? RunnerJobState.PENDING | ||
238 | : errorState | ||
239 | |||
240 | await this.specificError({ ...options, nextState }) | ||
241 | |||
242 | if (nextState === errorState) { | ||
243 | runnerJob.setToErrorOrCancel(nextState) | ||
244 | runnerJob.error = message | ||
245 | } else { | ||
246 | runnerJob.resetToPending() | ||
247 | } | ||
248 | |||
249 | await retryTransactionWrapper(() => { | ||
250 | return sequelizeTypescript.transaction(async transaction => { | ||
251 | await runnerJob.save({ transaction }) | ||
252 | }) | ||
253 | }) | ||
254 | |||
255 | if (runnerJob.state === errorState) { | ||
256 | const children = await RunnerJobModel.listChildrenOf(runnerJob) | ||
257 | |||
258 | for (const child of children) { | ||
259 | logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid)) | ||
260 | |||
261 | await this.error({ runnerJob: child, message: 'Parent error', fromParent: true }) | ||
262 | } | ||
263 | } | ||
264 | } | ||
265 | |||
266 | protected abstract specificError (options: { | ||
267 | runnerJob: MRunnerJob | ||
268 | message: string | ||
269 | nextState: RunnerJobState | ||
270 | }): Promise<void> | void | ||
271 | } | ||
diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts new file mode 100644 index 000000000..517645848 --- /dev/null +++ b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts | |||
@@ -0,0 +1,71 @@ | |||
1 | |||
2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' | ||
5 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
6 | import { MRunnerJob } from '@server/types/models/runners' | ||
7 | import { | ||
8 | LiveRTMPHLSTranscodingUpdatePayload, | ||
9 | RunnerJobSuccessPayload, | ||
10 | RunnerJobUpdatePayload, | ||
11 | RunnerJobVODPrivatePayload | ||
12 | } from '@shared/models' | ||
13 | import { AbstractJobHandler } from './abstract-job-handler' | ||
14 | import { loadTranscodingRunnerVideo } from './shared' | ||
15 | |||
16 | // eslint-disable-next-line max-len | ||
17 | export abstract class AbstractVODTranscodingJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> extends AbstractJobHandler<C, U, S> { | ||
18 | |||
19 | // --------------------------------------------------------------------------- | ||
20 | |||
21 | protected isAbortSupported () { | ||
22 | return true | ||
23 | } | ||
24 | |||
25 | protected specificUpdate (_options: { | ||
26 | runnerJob: MRunnerJob | ||
27 | updatePayload?: LiveRTMPHLSTranscodingUpdatePayload | ||
28 | }) { | ||
29 | // empty | ||
30 | } | ||
31 | |||
32 | protected specificAbort (_options: { | ||
33 | runnerJob: MRunnerJob | ||
34 | }) { | ||
35 | // empty | ||
36 | } | ||
37 | |||
38 | protected async specificError (options: { | ||
39 | runnerJob: MRunnerJob | ||
40 | }) { | ||
41 | const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) | ||
42 | if (!video) return | ||
43 | |||
44 | await moveToFailedTranscodingState(video) | ||
45 | |||
46 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
47 | } | ||
48 | |||
49 | protected async specificCancel (options: { | ||
50 | runnerJob: MRunnerJob | ||
51 | }) { | ||
52 | const { runnerJob } = options | ||
53 | |||
54 | const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) | ||
55 | if (!video) return | ||
56 | |||
57 | const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
58 | |||
59 | logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid)) | ||
60 | |||
61 | if (pending === 0) { | ||
62 | logger.info( | ||
63 | `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`, | ||
64 | this.lTags(video.uuid) | ||
65 | ) | ||
66 | |||
67 | const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload | ||
68 | await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo }) | ||
69 | } | ||
70 | } | ||
71 | } | ||
diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts new file mode 100644 index 000000000..0fca72b9a --- /dev/null +++ b/server/lib/runners/job-handlers/index.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export * from './abstract-job-handler' | ||
2 | export * from './live-rtmp-hls-transcoding-job-handler' | ||
3 | export * from './vod-audio-merge-transcoding-job-handler' | ||
4 | export * from './vod-hls-transcoding-job-handler' | ||
5 | export * from './vod-web-video-transcoding-job-handler' | ||
6 | export * from './runner-job-handlers' | ||
diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts new file mode 100644 index 000000000..c3d0e427d --- /dev/null +++ b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts | |||
@@ -0,0 +1,170 @@ | |||
1 | import { move, remove } from 'fs-extra' | ||
2 | import { join } from 'path' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { JOB_PRIORITY } from '@server/initializers/constants' | ||
5 | import { LiveManager } from '@server/lib/live' | ||
6 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | ||
7 | import { MRunnerJob } from '@server/types/models/runners' | ||
8 | import { buildUUID } from '@shared/extra-utils' | ||
9 | import { | ||
10 | LiveRTMPHLSTranscodingSuccess, | ||
11 | LiveRTMPHLSTranscodingUpdatePayload, | ||
12 | LiveVideoError, | ||
13 | RunnerJobLiveRTMPHLSTranscodingPayload, | ||
14 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload, | ||
15 | RunnerJobState | ||
16 | } from '@shared/models' | ||
17 | import { AbstractJobHandler } from './abstract-job-handler' | ||
18 | |||
19 | type CreateOptions = { | ||
20 | video: MVideo | ||
21 | playlist: MStreamingPlaylist | ||
22 | |||
23 | rtmpUrl: string | ||
24 | |||
25 | toTranscode: { | ||
26 | resolution: number | ||
27 | fps: number | ||
28 | }[] | ||
29 | |||
30 | segmentListSize: number | ||
31 | segmentDuration: number | ||
32 | |||
33 | outputDirectory: string | ||
34 | } | ||
35 | |||
36 | // eslint-disable-next-line max-len | ||
37 | export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateOptions, LiveRTMPHLSTranscodingUpdatePayload, LiveRTMPHLSTranscodingSuccess> { | ||
38 | |||
39 | async create (options: CreateOptions) { | ||
40 | const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory } = options | ||
41 | |||
42 | const jobUUID = buildUUID() | ||
43 | const payload: RunnerJobLiveRTMPHLSTranscodingPayload = { | ||
44 | input: { | ||
45 | rtmpUrl | ||
46 | }, | ||
47 | output: { | ||
48 | toTranscode, | ||
49 | segmentListSize, | ||
50 | segmentDuration | ||
51 | } | ||
52 | } | ||
53 | |||
54 | const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = { | ||
55 | videoUUID: video.uuid, | ||
56 | masterPlaylistName: playlist.playlistFilename, | ||
57 | outputDirectory | ||
58 | } | ||
59 | |||
60 | const job = await this.createRunnerJob({ | ||
61 | type: 'live-rtmp-hls-transcoding', | ||
62 | jobUUID, | ||
63 | payload, | ||
64 | privatePayload, | ||
65 | priority: JOB_PRIORITY.TRANSCODING | ||
66 | }) | ||
67 | |||
68 | return job | ||
69 | } | ||
70 | |||
71 | // --------------------------------------------------------------------------- | ||
72 | |||
73 | async specificUpdate (options: { | ||
74 | runnerJob: MRunnerJob | ||
75 | updatePayload: LiveRTMPHLSTranscodingUpdatePayload | ||
76 | }) { | ||
77 | const { runnerJob, updatePayload } = options | ||
78 | |||
79 | const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
80 | const outputDirectory = privatePayload.outputDirectory | ||
81 | const videoUUID = privatePayload.videoUUID | ||
82 | |||
83 | if (updatePayload.type === 'add-chunk') { | ||
84 | await move( | ||
85 | updatePayload.videoChunkFile as string, | ||
86 | join(outputDirectory, updatePayload.videoChunkFilename), | ||
87 | { overwrite: true } | ||
88 | ) | ||
89 | } else if (updatePayload.type === 'remove-chunk') { | ||
90 | await remove(join(outputDirectory, updatePayload.videoChunkFilename)) | ||
91 | } | ||
92 | |||
93 | if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) { | ||
94 | await move( | ||
95 | updatePayload.resolutionPlaylistFile as string, | ||
96 | join(outputDirectory, updatePayload.resolutionPlaylistFilename), | ||
97 | { overwrite: true } | ||
98 | ) | ||
99 | } | ||
100 | |||
101 | if (updatePayload.masterPlaylistFile) { | ||
102 | await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true }) | ||
103 | } | ||
104 | |||
105 | logger.info( | ||
106 | 'Runner live RTMP to HLS job %s for %s updated.', | ||
107 | runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) } | ||
108 | ) | ||
109 | } | ||
110 | |||
111 | // --------------------------------------------------------------------------- | ||
112 | |||
113 | protected specificComplete (options: { | ||
114 | runnerJob: MRunnerJob | ||
115 | }) { | ||
116 | return this.stopLive({ | ||
117 | runnerJob: options.runnerJob, | ||
118 | type: 'ended' | ||
119 | }) | ||
120 | } | ||
121 | |||
122 | // --------------------------------------------------------------------------- | ||
123 | |||
124 | protected isAbortSupported () { | ||
125 | return false | ||
126 | } | ||
127 | |||
128 | protected specificAbort () { | ||
129 | throw new Error('Not implemented') | ||
130 | } | ||
131 | |||
132 | protected specificError (options: { | ||
133 | runnerJob: MRunnerJob | ||
134 | nextState: RunnerJobState | ||
135 | }) { | ||
136 | return this.stopLive({ | ||
137 | runnerJob: options.runnerJob, | ||
138 | type: 'errored' | ||
139 | }) | ||
140 | } | ||
141 | |||
142 | protected specificCancel (options: { | ||
143 | runnerJob: MRunnerJob | ||
144 | }) { | ||
145 | return this.stopLive({ | ||
146 | runnerJob: options.runnerJob, | ||
147 | type: 'cancelled' | ||
148 | }) | ||
149 | } | ||
150 | |||
151 | private stopLive (options: { | ||
152 | runnerJob: MRunnerJob | ||
153 | type: 'ended' | 'errored' | 'cancelled' | ||
154 | }) { | ||
155 | const { runnerJob, type } = options | ||
156 | |||
157 | const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
158 | const videoUUID = privatePayload.videoUUID | ||
159 | |||
160 | const errorType = { | ||
161 | ended: null, | ||
162 | errored: LiveVideoError.RUNNER_JOB_ERROR, | ||
163 | cancelled: LiveVideoError.RUNNER_JOB_CANCEL | ||
164 | } | ||
165 | |||
166 | LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type]) | ||
167 | |||
168 | logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID)) | ||
169 | } | ||
170 | } | ||
diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts new file mode 100644 index 000000000..7bad1bc77 --- /dev/null +++ b/server/lib/runners/job-handlers/runner-job-handlers.ts | |||
@@ -0,0 +1,18 @@ | |||
1 | import { MRunnerJob } from '@server/types/models/runners' | ||
2 | import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' | ||
3 | import { AbstractJobHandler } from './abstract-job-handler' | ||
4 | import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' | ||
5 | import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' | ||
6 | import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' | ||
7 | import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' | ||
8 | |||
9 | const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = { | ||
10 | 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, | ||
11 | 'vod-hls-transcoding': VODHLSTranscodingJobHandler, | ||
12 | 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, | ||
13 | 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler | ||
14 | } | ||
15 | |||
16 | export function getRunnerJobHandlerClass (job: MRunnerJob) { | ||
17 | return processors[job.type] | ||
18 | } | ||
diff --git a/server/lib/runners/job-handlers/shared/index.ts b/server/lib/runners/job-handlers/shared/index.ts new file mode 100644 index 000000000..348273ae2 --- /dev/null +++ b/server/lib/runners/job-handlers/shared/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './vod-helpers' | |||
diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts new file mode 100644 index 000000000..93ae89ff8 --- /dev/null +++ b/server/lib/runners/job-handlers/shared/vod-helpers.ts | |||
@@ -0,0 +1,44 @@ | |||
1 | import { move } from 'fs-extra' | ||
2 | import { dirname, join } from 'path' | ||
3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | ||
4 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | ||
5 | import { onWebTorrentVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' | ||
6 | import { buildNewFile } from '@server/lib/video-file' | ||
7 | import { VideoModel } from '@server/models/video/video' | ||
8 | import { MVideoFullLight } from '@server/types/models' | ||
9 | import { MRunnerJob } from '@server/types/models/runners' | ||
10 | import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models' | ||
11 | |||
12 | export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { | ||
13 | video: MVideoFullLight | ||
14 | videoFilePath: string | ||
15 | privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload | ||
16 | }) { | ||
17 | const { video, videoFilePath, privatePayload } = options | ||
18 | |||
19 | const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' }) | ||
20 | videoFile.videoId = video.id | ||
21 | |||
22 | const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) | ||
23 | await move(videoFilePath, newVideoFilePath) | ||
24 | |||
25 | await onWebTorrentVideoFileTranscoding({ | ||
26 | video, | ||
27 | videoFile, | ||
28 | videoOutputPath: newVideoFilePath | ||
29 | }) | ||
30 | |||
31 | await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) | ||
32 | } | ||
33 | |||
34 | export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) { | ||
35 | const videoUUID = runnerJob.privatePayload.videoUUID | ||
36 | |||
37 | const video = await VideoModel.loadFull(videoUUID) | ||
38 | if (!video) { | ||
39 | logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID)) | ||
40 | return undefined | ||
41 | } | ||
42 | |||
43 | return video | ||
44 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts new file mode 100644 index 000000000..a7b33f87e --- /dev/null +++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts | |||
@@ -0,0 +1,97 @@ | |||
1 | import { pick } from 'lodash' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
4 | import { MVideo } from '@server/types/models' | ||
5 | import { MRunnerJob } from '@server/types/models/runners' | ||
6 | import { buildUUID } from '@shared/extra-utils' | ||
7 | import { getVideoStreamDuration } from '@shared/ffmpeg' | ||
8 | import { | ||
9 | RunnerJobUpdatePayload, | ||
10 | RunnerJobVODAudioMergeTranscodingPayload, | ||
11 | RunnerJobVODWebVideoTranscodingPrivatePayload, | ||
12 | VODAudioMergeTranscodingSuccess | ||
13 | } from '@shared/models' | ||
14 | import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls' | ||
15 | import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' | ||
16 | import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' | ||
17 | |||
18 | type CreateOptions = { | ||
19 | video: MVideo | ||
20 | isNewVideo: boolean | ||
21 | resolution: number | ||
22 | fps: number | ||
23 | priority: number | ||
24 | dependsOnRunnerJob?: MRunnerJob | ||
25 | } | ||
26 | |||
27 | // eslint-disable-next-line max-len | ||
28 | export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODAudioMergeTranscodingSuccess> { | ||
29 | |||
30 | async create (options: CreateOptions) { | ||
31 | const { video, resolution, fps, priority, dependsOnRunnerJob } = options | ||
32 | |||
33 | const jobUUID = buildUUID() | ||
34 | const payload: RunnerJobVODAudioMergeTranscodingPayload = { | ||
35 | input: { | ||
36 | audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid), | ||
37 | previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid) | ||
38 | }, | ||
39 | output: { | ||
40 | resolution, | ||
41 | fps | ||
42 | } | ||
43 | } | ||
44 | |||
45 | const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { | ||
46 | ...pick(options, [ 'isNewVideo' ]), | ||
47 | |||
48 | videoUUID: video.uuid | ||
49 | } | ||
50 | |||
51 | const job = await this.createRunnerJob({ | ||
52 | type: 'vod-audio-merge-transcoding', | ||
53 | jobUUID, | ||
54 | payload, | ||
55 | privatePayload, | ||
56 | priority, | ||
57 | dependsOnRunnerJob | ||
58 | }) | ||
59 | |||
60 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
61 | |||
62 | return job | ||
63 | } | ||
64 | |||
65 | // --------------------------------------------------------------------------- | ||
66 | |||
67 | async specificComplete (options: { | ||
68 | runnerJob: MRunnerJob | ||
69 | resultPayload: VODAudioMergeTranscodingSuccess | ||
70 | }) { | ||
71 | const { runnerJob, resultPayload } = options | ||
72 | const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload | ||
73 | |||
74 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
75 | if (!video) return | ||
76 | |||
77 | const videoFilePath = resultPayload.videoFile as string | ||
78 | |||
79 | // ffmpeg generated a new video file, so update the video duration | ||
80 | // See https://trac.ffmpeg.org/ticket/5456 | ||
81 | video.duration = await getVideoStreamDuration(videoFilePath) | ||
82 | await video.save() | ||
83 | |||
84 | // We can remove the old audio file | ||
85 | const oldAudioFile = video.VideoFiles[0] | ||
86 | await video.removeWebTorrentFile(oldAudioFile) | ||
87 | await oldAudioFile.destroy() | ||
88 | video.VideoFiles = [] | ||
89 | |||
90 | await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) | ||
91 | |||
92 | logger.info( | ||
93 | 'Runner VOD audio merge transcoding job %s for %s ended.', | ||
94 | runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) | ||
95 | ) | ||
96 | } | ||
97 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts new file mode 100644 index 000000000..02566b9d5 --- /dev/null +++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts | |||
@@ -0,0 +1,114 @@ | |||
1 | import { move } from 'fs-extra' | ||
2 | import { dirname, join } from 'path' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { renameVideoFileInPlaylist } from '@server/lib/hls' | ||
5 | import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' | ||
6 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | ||
7 | import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' | ||
8 | import { buildNewFile, removeAllWebTorrentFiles } from '@server/lib/video-file' | ||
9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
10 | import { MVideo } from '@server/types/models' | ||
11 | import { MRunnerJob } from '@server/types/models/runners' | ||
12 | import { pick } from '@shared/core-utils' | ||
13 | import { buildUUID } from '@shared/extra-utils' | ||
14 | import { | ||
15 | RunnerJobUpdatePayload, | ||
16 | RunnerJobVODHLSTranscodingPayload, | ||
17 | RunnerJobVODHLSTranscodingPrivatePayload, | ||
18 | VODHLSTranscodingSuccess | ||
19 | } from '@shared/models' | ||
20 | import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' | ||
21 | import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' | ||
22 | import { loadTranscodingRunnerVideo } from './shared' | ||
23 | |||
24 | type CreateOptions = { | ||
25 | video: MVideo | ||
26 | isNewVideo: boolean | ||
27 | deleteWebVideoFiles: boolean | ||
28 | resolution: number | ||
29 | fps: number | ||
30 | priority: number | ||
31 | dependsOnRunnerJob?: MRunnerJob | ||
32 | } | ||
33 | |||
34 | // eslint-disable-next-line max-len | ||
35 | export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODHLSTranscodingSuccess> { | ||
36 | |||
37 | async create (options: CreateOptions) { | ||
38 | const { video, resolution, fps, dependsOnRunnerJob, priority } = options | ||
39 | |||
40 | const jobUUID = buildUUID() | ||
41 | |||
42 | const payload: RunnerJobVODHLSTranscodingPayload = { | ||
43 | input: { | ||
44 | videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) | ||
45 | }, | ||
46 | output: { | ||
47 | resolution, | ||
48 | fps | ||
49 | } | ||
50 | } | ||
51 | |||
52 | const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = { | ||
53 | ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]), | ||
54 | |||
55 | videoUUID: video.uuid | ||
56 | } | ||
57 | |||
58 | const job = await this.createRunnerJob({ | ||
59 | type: 'vod-hls-transcoding', | ||
60 | jobUUID, | ||
61 | payload, | ||
62 | privatePayload, | ||
63 | priority, | ||
64 | dependsOnRunnerJob | ||
65 | }) | ||
66 | |||
67 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
68 | |||
69 | return job | ||
70 | } | ||
71 | |||
72 | // --------------------------------------------------------------------------- | ||
73 | |||
74 | async specificComplete (options: { | ||
75 | runnerJob: MRunnerJob | ||
76 | resultPayload: VODHLSTranscodingSuccess | ||
77 | }) { | ||
78 | const { runnerJob, resultPayload } = options | ||
79 | const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload | ||
80 | |||
81 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
82 | if (!video) return | ||
83 | |||
84 | const videoFilePath = resultPayload.videoFile as string | ||
85 | const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string | ||
86 | |||
87 | const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' }) | ||
88 | const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) | ||
89 | await move(videoFilePath, newVideoFilePath) | ||
90 | |||
91 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename) | ||
92 | const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename) | ||
93 | await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath) | ||
94 | |||
95 | await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename) | ||
96 | |||
97 | await onHLSVideoFileTranscoding({ | ||
98 | video, | ||
99 | videoFile, | ||
100 | m3u8OutputPath: newResolutionPlaylistFilePath, | ||
101 | videoOutputPath: newVideoFilePath | ||
102 | }) | ||
103 | |||
104 | await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) | ||
105 | |||
106 | if (privatePayload.deleteWebVideoFiles === true) { | ||
107 | logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) | ||
108 | |||
109 | await removeAllWebTorrentFiles(video) | ||
110 | } | ||
111 | |||
112 | logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) | ||
113 | } | ||
114 | } | ||
diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts new file mode 100644 index 000000000..57761a7a1 --- /dev/null +++ b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts | |||
@@ -0,0 +1,84 @@ | |||
1 | import { pick } from 'lodash' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
4 | import { MVideo } from '@server/types/models' | ||
5 | import { MRunnerJob } from '@server/types/models/runners' | ||
6 | import { buildUUID } from '@shared/extra-utils' | ||
7 | import { | ||
8 | RunnerJobUpdatePayload, | ||
9 | RunnerJobVODWebVideoTranscodingPayload, | ||
10 | RunnerJobVODWebVideoTranscodingPrivatePayload, | ||
11 | VODWebVideoTranscodingSuccess | ||
12 | } from '@shared/models' | ||
13 | import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' | ||
14 | import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' | ||
15 | import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' | ||
16 | |||
17 | type CreateOptions = { | ||
18 | video: MVideo | ||
19 | isNewVideo: boolean | ||
20 | resolution: number | ||
21 | fps: number | ||
22 | priority: number | ||
23 | dependsOnRunnerJob?: MRunnerJob | ||
24 | } | ||
25 | |||
26 | // eslint-disable-next-line max-len | ||
27 | export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODWebVideoTranscodingSuccess> { | ||
28 | |||
29 | async create (options: CreateOptions) { | ||
30 | const { video, resolution, fps, priority, dependsOnRunnerJob } = options | ||
31 | |||
32 | const jobUUID = buildUUID() | ||
33 | const payload: RunnerJobVODWebVideoTranscodingPayload = { | ||
34 | input: { | ||
35 | videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) | ||
36 | }, | ||
37 | output: { | ||
38 | resolution, | ||
39 | fps | ||
40 | } | ||
41 | } | ||
42 | |||
43 | const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { | ||
44 | ...pick(options, [ 'isNewVideo' ]), | ||
45 | |||
46 | videoUUID: video.uuid | ||
47 | } | ||
48 | |||
49 | const job = await this.createRunnerJob({ | ||
50 | type: 'vod-web-video-transcoding', | ||
51 | jobUUID, | ||
52 | payload, | ||
53 | privatePayload, | ||
54 | dependsOnRunnerJob, | ||
55 | priority | ||
56 | }) | ||
57 | |||
58 | await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') | ||
59 | |||
60 | return job | ||
61 | } | ||
62 | |||
63 | // --------------------------------------------------------------------------- | ||
64 | |||
65 | async specificComplete (options: { | ||
66 | runnerJob: MRunnerJob | ||
67 | resultPayload: VODWebVideoTranscodingSuccess | ||
68 | }) { | ||
69 | const { runnerJob, resultPayload } = options | ||
70 | const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload | ||
71 | |||
72 | const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) | ||
73 | if (!video) return | ||
74 | |||
75 | const videoFilePath = resultPayload.videoFile as string | ||
76 | |||
77 | await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) | ||
78 | |||
79 | logger.info( | ||
80 | 'Runner VOD web video transcoding job %s for %s ended.', | ||
81 | runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) | ||
82 | ) | ||
83 | } | ||
84 | } | ||
diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts new file mode 100644 index 000000000..329fb1170 --- /dev/null +++ b/server/lib/runners/runner-urls.ts | |||
@@ -0,0 +1,9 @@ | |||
1 | import { WEBSERVER } from '@server/initializers/constants' | ||
2 | |||
3 | export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) { | ||
4 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality' | ||
5 | } | ||
6 | |||
7 | export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { | ||
8 | return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' | ||
9 | } | ||
diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts new file mode 100644 index 000000000..74c814ba1 --- /dev/null +++ b/server/lib/runners/runner.ts | |||
@@ -0,0 +1,36 @@ | |||
1 | import express from 'express' | ||
2 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
4 | import { sequelizeTypescript } from '@server/initializers/database' | ||
5 | import { MRunner } from '@server/types/models/runners' | ||
6 | |||
7 | const lTags = loggerTagsFactory('runner') | ||
8 | |||
9 | const updatingRunner = new Set<number>() | ||
10 | |||
11 | function updateLastRunnerContact (req: express.Request, runner: MRunner) { | ||
12 | const now = new Date() | ||
13 | |||
14 | // Don't update last runner contact too often | ||
15 | if (now.getTime() - runner.lastContact.getTime() < 2000) return | ||
16 | if (updatingRunner.has(runner.id)) return | ||
17 | |||
18 | updatingRunner.add(runner.id) | ||
19 | |||
20 | runner.lastContact = now | ||
21 | runner.ip = req.ip | ||
22 | |||
23 | logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name)) | ||
24 | |||
25 | retryTransactionWrapper(() => { | ||
26 | return sequelizeTypescript.transaction(async transaction => { | ||
27 | return runner.save({ transaction }) | ||
28 | }) | ||
29 | }) | ||
30 | .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) })) | ||
31 | .finally(() => updatingRunner.delete(runner.id)) | ||
32 | } | ||
33 | |||
34 | export { | ||
35 | updateLastRunnerContact | ||
36 | } | ||
diff --git a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts new file mode 100644 index 000000000..f7a26d2bc --- /dev/null +++ b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts | |||
@@ -0,0 +1,42 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
3 | import { logger, loggerTagsFactory } from '../../helpers/logger' | ||
4 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
5 | import { getRunnerJobHandlerClass } from '../runners' | ||
6 | import { AbstractScheduler } from './abstract-scheduler' | ||
7 | |||
8 | const lTags = loggerTagsFactory('runner') | ||
9 | |||
10 | export class RunnerJobWatchDogScheduler extends AbstractScheduler { | ||
11 | |||
12 | private static instance: AbstractScheduler | ||
13 | |||
14 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG | ||
15 | |||
16 | private constructor () { | ||
17 | super() | ||
18 | } | ||
19 | |||
20 | protected async internalExecute () { | ||
21 | const vodStalledJobs = await RunnerJobModel.listStalledJobs({ | ||
22 | staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, | ||
23 | types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ] | ||
24 | }) | ||
25 | |||
26 | const liveStalledJobs = await RunnerJobModel.listStalledJobs({ | ||
27 | staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE, | ||
28 | types: [ 'live-rtmp-hls-transcoding' ] | ||
29 | }) | ||
30 | |||
31 | for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { | ||
32 | logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) | ||
33 | |||
34 | const Handler = getRunnerJobHandlerClass(stalled) | ||
35 | await new Handler().abort({ runnerJob: stalled }) | ||
36 | } | ||
37 | } | ||
38 | |||
39 | static get Instance () { | ||
40 | return this.instance || (this.instance = new this()) | ||
41 | } | ||
42 | } | ||
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index e87e2854f..ba7916363 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts | |||
@@ -126,11 +126,14 @@ class ServerConfigManager { | |||
126 | serverVersion: PEERTUBE_VERSION, | 126 | serverVersion: PEERTUBE_VERSION, |
127 | serverCommit: this.serverCommit, | 127 | serverCommit: this.serverCommit, |
128 | transcoding: { | 128 | transcoding: { |
129 | remoteRunners: { | ||
130 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
131 | }, | ||
129 | hls: { | 132 | hls: { |
130 | enabled: CONFIG.TRANSCODING.HLS.ENABLED | 133 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED |
131 | }, | 134 | }, |
132 | webtorrent: { | 135 | webtorrent: { |
133 | enabled: CONFIG.TRANSCODING.WEBTORRENT.ENABLED | 136 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEBTORRENT.ENABLED |
134 | }, | 137 | }, |
135 | enabledResolutions: this.getEnabledResolutions('vod'), | 138 | enabledResolutions: this.getEnabledResolutions('vod'), |
136 | profile: CONFIG.TRANSCODING.PROFILE, | 139 | profile: CONFIG.TRANSCODING.PROFILE, |
@@ -150,6 +153,9 @@ class ServerConfigManager { | |||
150 | 153 | ||
151 | transcoding: { | 154 | transcoding: { |
152 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, | 155 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED, |
156 | remoteRunners: { | ||
157 | enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED | ||
158 | }, | ||
153 | enabledResolutions: this.getEnabledResolutions('live'), | 159 | enabledResolutions: this.getEnabledResolutions('live'), |
154 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, | 160 | profile: CONFIG.LIVE.TRANSCODING.PROFILE, |
155 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') | 161 | availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') |
diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts new file mode 100644 index 000000000..46831a912 --- /dev/null +++ b/server/lib/transcoding/create-transcoding-job.ts | |||
@@ -0,0 +1,36 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
3 | import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared' | ||
4 | |||
5 | export function createOptimizeOrMergeAudioJobs (options: { | ||
6 | video: MVideoFullLight | ||
7 | videoFile: MVideoFile | ||
8 | isNewVideo: boolean | ||
9 | user: MUserId | ||
10 | }) { | ||
11 | return getJobBuilder().createOptimizeOrMergeAudioJobs(options) | ||
12 | } | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export function createTranscodingJobs (options: { | ||
17 | transcodingType: 'hls' | 'webtorrent' | ||
18 | video: MVideoFullLight | ||
19 | resolutions: number[] | ||
20 | isNewVideo: boolean | ||
21 | user: MUserId | ||
22 | }) { | ||
23 | return getJobBuilder().createTranscodingJobs(options) | ||
24 | } | ||
25 | |||
26 | // --------------------------------------------------------------------------- | ||
27 | // Private | ||
28 | // --------------------------------------------------------------------------- | ||
29 | |||
30 | function getJobBuilder () { | ||
31 | if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) { | ||
32 | return new TranscodingRunnerJobBuilder() | ||
33 | } | ||
34 | |||
35 | return new TranscodingJobQueueBuilder() | ||
36 | } | ||
diff --git a/server/lib/transcoding/default-transcoding-profiles.ts b/server/lib/transcoding/default-transcoding-profiles.ts index f47718819..5251784ac 100644 --- a/server/lib/transcoding/default-transcoding-profiles.ts +++ b/server/lib/transcoding/default-transcoding-profiles.ts | |||
@@ -1,15 +1,9 @@ | |||
1 | 1 | ||
2 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
3 | import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils' | 3 | import { getAverageBitrate, getMinLimitBitrate } from '@shared/core-utils' |
4 | import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '../../../shared/models/videos' | 4 | import { buildStreamSuffix, FFmpegCommandWrapper, ffprobePromise, getAudioStream, getMaxAudioBitrate } from '@shared/ffmpeg' |
5 | import { | 5 | import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models' |
6 | buildStreamSuffix, | 6 | import { canDoQuickAudioTranscode } from './transcoding-quick-transcode' |
7 | canDoQuickAudioTranscode, | ||
8 | ffprobePromise, | ||
9 | getAudioStream, | ||
10 | getMaxAudioBitrate, | ||
11 | resetSupportedEncoders | ||
12 | } from '../../helpers/ffmpeg' | ||
13 | 7 | ||
14 | /** | 8 | /** |
15 | * | 9 | * |
@@ -184,14 +178,14 @@ class VideoTranscodingProfilesManager { | |||
184 | addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { | 178 | addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { |
185 | this.encodersPriorities[type][streamType].push({ name: encoder, priority }) | 179 | this.encodersPriorities[type][streamType].push({ name: encoder, priority }) |
186 | 180 | ||
187 | resetSupportedEncoders() | 181 | FFmpegCommandWrapper.resetSupportedEncoders() |
188 | } | 182 | } |
189 | 183 | ||
190 | removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { | 184 | removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { |
191 | this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] | 185 | this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] |
192 | .filter(o => o.name !== encoder && o.priority !== priority) | 186 | .filter(o => o.name !== encoder && o.priority !== priority) |
193 | 187 | ||
194 | resetSupportedEncoders() | 188 | FFmpegCommandWrapper.resetSupportedEncoders() |
195 | } | 189 | } |
196 | 190 | ||
197 | private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { | 191 | private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { |
diff --git a/server/lib/transcoding/ended-transcoding.ts b/server/lib/transcoding/ended-transcoding.ts new file mode 100644 index 000000000..d31674ede --- /dev/null +++ b/server/lib/transcoding/ended-transcoding.ts | |||
@@ -0,0 +1,18 @@ | |||
1 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
2 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
3 | import { MVideo } from '@server/types/models' | ||
4 | import { moveToNextState } from '../video-state' | ||
5 | |||
6 | export async function onTranscodingEnded (options: { | ||
7 | video: MVideo | ||
8 | isNewVideo: boolean | ||
9 | moveVideoToNextState: boolean | ||
10 | }) { | ||
11 | const { video, isNewVideo, moveVideoToNextState } = options | ||
12 | |||
13 | await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') | ||
14 | |||
15 | if (moveVideoToNextState) { | ||
16 | await retryTransactionWrapper(moveToNextState, { video, isNewVideo }) | ||
17 | } | ||
18 | } | ||
diff --git a/server/lib/transcoding/hls-transcoding.ts b/server/lib/transcoding/hls-transcoding.ts new file mode 100644 index 000000000..cffa859c7 --- /dev/null +++ b/server/lib/transcoding/hls-transcoding.ts | |||
@@ -0,0 +1,181 @@ | |||
1 | import { MutexInterface } from 'async-mutex' | ||
2 | import { Job } from 'bullmq' | ||
3 | import { ensureDir, move, stat } from 'fs-extra' | ||
4 | import { basename, extname as extnameUtil, join } from 'path' | ||
5 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
6 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
7 | import { sequelizeTypescript } from '@server/initializers/database' | ||
8 | import { MVideo, MVideoFile } from '@server/types/models' | ||
9 | import { pick } from '@shared/core-utils' | ||
10 | import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' | ||
11 | import { VideoResolution } from '@shared/models' | ||
12 | import { CONFIG } from '../../initializers/config' | ||
13 | import { VideoFileModel } from '../../models/video/video-file' | ||
14 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
15 | import { updatePlaylistAfterFileChange } from '../hls' | ||
16 | import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' | ||
17 | import { buildFileMetadata } from '../video-file' | ||
18 | import { VideoPathManager } from '../video-path-manager' | ||
19 | import { buildFFmpegVOD } from './shared' | ||
20 | |||
21 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | ||
22 | export async function generateHlsPlaylistResolutionFromTS (options: { | ||
23 | video: MVideo | ||
24 | concatenatedTsFilePath: string | ||
25 | resolution: VideoResolution | ||
26 | fps: number | ||
27 | isAAC: boolean | ||
28 | inputFileMutexReleaser: MutexInterface.Releaser | ||
29 | }) { | ||
30 | return generateHlsPlaylistCommon({ | ||
31 | type: 'hls-from-ts' as 'hls-from-ts', | ||
32 | inputPath: options.concatenatedTsFilePath, | ||
33 | |||
34 | ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ]) | ||
35 | }) | ||
36 | } | ||
37 | |||
38 | // Generate an HLS playlist from an input file, and update the master playlist | ||
39 | export function generateHlsPlaylistResolution (options: { | ||
40 | video: MVideo | ||
41 | videoInputPath: string | ||
42 | resolution: VideoResolution | ||
43 | fps: number | ||
44 | copyCodecs: boolean | ||
45 | inputFileMutexReleaser: MutexInterface.Releaser | ||
46 | job?: Job | ||
47 | }) { | ||
48 | return generateHlsPlaylistCommon({ | ||
49 | type: 'hls' as 'hls', | ||
50 | inputPath: options.videoInputPath, | ||
51 | |||
52 | ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) | ||
53 | }) | ||
54 | } | ||
55 | |||
56 | export async function onHLSVideoFileTranscoding (options: { | ||
57 | video: MVideo | ||
58 | videoFile: MVideoFile | ||
59 | videoOutputPath: string | ||
60 | m3u8OutputPath: string | ||
61 | }) { | ||
62 | const { video, videoFile, videoOutputPath, m3u8OutputPath } = options | ||
63 | |||
64 | // Create or update the playlist | ||
65 | const playlist = await retryTransactionWrapper(() => { | ||
66 | return sequelizeTypescript.transaction(async transaction => { | ||
67 | return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) | ||
68 | }) | ||
69 | }) | ||
70 | videoFile.videoStreamingPlaylistId = playlist.id | ||
71 | |||
72 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
73 | |||
74 | try { | ||
75 | // VOD transcoding is a long task, refresh video attributes | ||
76 | await video.reload() | ||
77 | |||
78 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile) | ||
79 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | ||
80 | |||
81 | // Move playlist file | ||
82 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath)) | ||
83 | await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true }) | ||
84 | // Move video file | ||
85 | await move(videoOutputPath, videoFilePath, { overwrite: true }) | ||
86 | |||
87 | // Update video duration if it was not set (in case of a live for example) | ||
88 | if (!video.duration) { | ||
89 | video.duration = await getVideoStreamDuration(videoFilePath) | ||
90 | await video.save() | ||
91 | } | ||
92 | |||
93 | const stats = await stat(videoFilePath) | ||
94 | |||
95 | videoFile.size = stats.size | ||
96 | videoFile.fps = await getVideoStreamFPS(videoFilePath) | ||
97 | videoFile.metadata = await buildFileMetadata(videoFilePath) | ||
98 | |||
99 | await createTorrentAndSetInfoHash(playlist, videoFile) | ||
100 | |||
101 | const oldFile = await VideoFileModel.loadHLSFile({ | ||
102 | playlistId: playlist.id, | ||
103 | fps: videoFile.fps, | ||
104 | resolution: videoFile.resolution | ||
105 | }) | ||
106 | |||
107 | if (oldFile) { | ||
108 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | ||
109 | await oldFile.destroy() | ||
110 | } | ||
111 | |||
112 | const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined) | ||
113 | |||
114 | await updatePlaylistAfterFileChange(video, playlist) | ||
115 | |||
116 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | ||
117 | } finally { | ||
118 | mutexReleaser() | ||
119 | } | ||
120 | } | ||
121 | |||
122 | // --------------------------------------------------------------------------- | ||
123 | |||
124 | async function generateHlsPlaylistCommon (options: { | ||
125 | type: 'hls' | 'hls-from-ts' | ||
126 | video: MVideo | ||
127 | inputPath: string | ||
128 | |||
129 | resolution: VideoResolution | ||
130 | fps: number | ||
131 | |||
132 | inputFileMutexReleaser: MutexInterface.Releaser | ||
133 | |||
134 | copyCodecs?: boolean | ||
135 | isAAC?: boolean | ||
136 | |||
137 | job?: Job | ||
138 | }) { | ||
139 | const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options | ||
140 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
141 | |||
142 | const videoTranscodedBasePath = join(transcodeDirectory, type) | ||
143 | await ensureDir(videoTranscodedBasePath) | ||
144 | |||
145 | const videoFilename = generateHLSVideoFilename(resolution) | ||
146 | const videoOutputPath = join(videoTranscodedBasePath, videoFilename) | ||
147 | |||
148 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) | ||
149 | const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename) | ||
150 | |||
151 | const transcodeOptions = { | ||
152 | type, | ||
153 | |||
154 | inputPath, | ||
155 | outputPath: m3u8OutputPath, | ||
156 | |||
157 | resolution, | ||
158 | fps, | ||
159 | copyCodecs, | ||
160 | |||
161 | isAAC, | ||
162 | |||
163 | inputFileMutexReleaser, | ||
164 | |||
165 | hlsPlaylist: { | ||
166 | videoFilename | ||
167 | } | ||
168 | } | ||
169 | |||
170 | await buildFFmpegVOD(job).transcode(transcodeOptions) | ||
171 | |||
172 | const newVideoFile = new VideoFileModel({ | ||
173 | resolution, | ||
174 | extname: extnameUtil(videoFilename), | ||
175 | size: 0, | ||
176 | filename: videoFilename, | ||
177 | fps: -1 | ||
178 | }) | ||
179 | |||
180 | await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) | ||
181 | } | ||
diff --git a/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/lib/transcoding/shared/ffmpeg-builder.ts new file mode 100644 index 000000000..441445ec4 --- /dev/null +++ b/server/lib/transcoding/shared/ffmpeg-builder.ts | |||
@@ -0,0 +1,18 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { FFmpegVOD } from '@shared/ffmpeg' | ||
5 | import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles' | ||
6 | |||
7 | export function buildFFmpegVOD (job?: Job) { | ||
8 | return new FFmpegVOD({ | ||
9 | ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()), | ||
10 | |||
11 | updateJobProgress: progress => { | ||
12 | if (!job) return | ||
13 | |||
14 | job.updateProgress(progress) | ||
15 | .catch(err => logger.error('Cannot update ffmpeg job progress', { err })) | ||
16 | } | ||
17 | }) | ||
18 | } | ||
diff --git a/server/lib/transcoding/shared/index.ts b/server/lib/transcoding/shared/index.ts new file mode 100644 index 000000000..f0b45bcbb --- /dev/null +++ b/server/lib/transcoding/shared/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './job-builders' | ||
2 | export * from './ffmpeg-builder' | ||
diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts new file mode 100644 index 000000000..f1e9efdcf --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts | |||
@@ -0,0 +1,38 @@ | |||
1 | |||
2 | import { JOB_PRIORITY } from '@server/initializers/constants' | ||
3 | import { VideoModel } from '@server/models/video/video' | ||
4 | import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
5 | |||
6 | export abstract class AbstractJobBuilder { | ||
7 | |||
8 | abstract createOptimizeOrMergeAudioJobs (options: { | ||
9 | video: MVideoFullLight | ||
10 | videoFile: MVideoFile | ||
11 | isNewVideo: boolean | ||
12 | user: MUserId | ||
13 | }): Promise<any> | ||
14 | |||
15 | abstract createTranscodingJobs (options: { | ||
16 | transcodingType: 'hls' | 'webtorrent' | ||
17 | video: MVideoFullLight | ||
18 | resolutions: number[] | ||
19 | isNewVideo: boolean | ||
20 | user: MUserId | null | ||
21 | }): Promise<any> | ||
22 | |||
23 | protected async getTranscodingJobPriority (options: { | ||
24 | user: MUserId | ||
25 | fallback: number | ||
26 | }) { | ||
27 | const { user, fallback } = options | ||
28 | |||
29 | if (!user) return fallback | ||
30 | |||
31 | const now = new Date() | ||
32 | const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) | ||
33 | |||
34 | const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) | ||
35 | |||
36 | return JOB_PRIORITY.TRANSCODING + videoUploadedByUser | ||
37 | } | ||
38 | } | ||
diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts new file mode 100644 index 000000000..9b1c82adf --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './transcoding-job-queue-builder' | ||
2 | export * from './transcoding-runner-job-builder' | ||
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts new file mode 100644 index 000000000..7c892718b --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts | |||
@@ -0,0 +1,308 @@ | |||
1 | import Bluebird from 'bluebird' | ||
2 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
3 | import { logger } from '@server/helpers/logger' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
6 | import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' | ||
7 | import { Hooks } from '@server/lib/plugins/hooks' | ||
8 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
10 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' | ||
11 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' | ||
12 | import { | ||
13 | HLSTranscodingPayload, | ||
14 | MergeAudioTranscodingPayload, | ||
15 | NewWebTorrentResolutionTranscodingPayload, | ||
16 | OptimizeTranscodingPayload, | ||
17 | VideoTranscodingPayload | ||
18 | } from '@shared/models' | ||
19 | import { canDoQuickTranscode } from '../../transcoding-quick-transcode' | ||
20 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' | ||
21 | import { AbstractJobBuilder } from './abstract-job-builder' | ||
22 | |||
23 | export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | ||
24 | |||
25 | async createOptimizeOrMergeAudioJobs (options: { | ||
26 | video: MVideoFullLight | ||
27 | videoFile: MVideoFile | ||
28 | isNewVideo: boolean | ||
29 | user: MUserId | ||
30 | }) { | ||
31 | const { video, videoFile, isNewVideo, user } = options | ||
32 | |||
33 | let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload | ||
34 | let nextTranscodingSequentialJobPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | ||
35 | |||
36 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
37 | |||
38 | try { | ||
39 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { | ||
40 | const probe = await ffprobePromise(videoFilePath) | ||
41 | |||
42 | const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) | ||
43 | const hasAudio = await hasAudioStream(videoFilePath, probe) | ||
44 | const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) | ||
45 | const inputFPS = videoFile.isAudio() | ||
46 | ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value | ||
47 | : await getVideoStreamFPS(videoFilePath, probe) | ||
48 | |||
49 | const maxResolution = await isAudioFile(videoFilePath, probe) | ||
50 | ? DEFAULT_AUDIO_RESOLUTION | ||
51 | : resolution | ||
52 | |||
53 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | ||
54 | nextTranscodingSequentialJobPayloads.push([ | ||
55 | this.buildHLSJobPayload({ | ||
56 | deleteWebTorrentFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false, | ||
57 | |||
58 | // We had some issues with a web video quick transcoded while producing a HLS version of it | ||
59 | copyCodecs: !quickTranscode, | ||
60 | |||
61 | resolution: maxResolution, | ||
62 | fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), | ||
63 | videoUUID: video.uuid, | ||
64 | isNewVideo | ||
65 | }) | ||
66 | ]) | ||
67 | } | ||
68 | |||
69 | const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ | ||
70 | video, | ||
71 | inputVideoResolution: maxResolution, | ||
72 | inputVideoFPS: inputFPS, | ||
73 | hasAudio, | ||
74 | isNewVideo | ||
75 | }) | ||
76 | |||
77 | nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] | ||
78 | |||
79 | mergeOrOptimizePayload = videoFile.isAudio() | ||
80 | ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo }) | ||
81 | : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode }) | ||
82 | }) | ||
83 | } finally { | ||
84 | mutexReleaser() | ||
85 | } | ||
86 | |||
87 | const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { | ||
88 | return Bluebird.mapSeries(payloads, payload => { | ||
89 | return this.buildTranscodingJob({ payload, user }) | ||
90 | }) | ||
91 | }) | ||
92 | |||
93 | const transcodingJobBuilderJob: CreateJobArgument = { | ||
94 | type: 'transcoding-job-builder', | ||
95 | payload: { | ||
96 | videoUUID: video.uuid, | ||
97 | sequentialJobs: nextTranscodingSequentialJobs | ||
98 | } | ||
99 | } | ||
100 | |||
101 | const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) | ||
102 | |||
103 | return JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) | ||
104 | } | ||
105 | |||
106 | // --------------------------------------------------------------------------- | ||
107 | |||
108 | async createTranscodingJobs (options: { | ||
109 | transcodingType: 'hls' | 'webtorrent' | ||
110 | video: MVideoFullLight | ||
111 | resolutions: number[] | ||
112 | isNewVideo: boolean | ||
113 | user: MUserId | null | ||
114 | }) { | ||
115 | const { video, transcodingType, resolutions, isNewVideo } = options | ||
116 | |||
117 | const maxResolution = Math.max(...resolutions) | ||
118 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | ||
119 | |||
120 | logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) | ||
121 | |||
122 | const { fps: inputFPS } = await video.probeMaxQualityFile() | ||
123 | |||
124 | const children = childrenResolutions.map(resolution => { | ||
125 | const fps = computeOutputFPS({ inputFPS, resolution }) | ||
126 | |||
127 | if (transcodingType === 'hls') { | ||
128 | return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | ||
129 | } | ||
130 | |||
131 | if (transcodingType === 'webtorrent') { | ||
132 | return this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | ||
133 | } | ||
134 | |||
135 | throw new Error('Unknown transcoding type') | ||
136 | }) | ||
137 | |||
138 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
139 | |||
140 | const parent = transcodingType === 'hls' | ||
141 | ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | ||
142 | : this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | ||
143 | |||
144 | // Process the last resolution after the other ones to prevent concurrency issue | ||
145 | // Because low resolutions use the biggest one as ffmpeg input | ||
146 | await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) | ||
147 | } | ||
148 | |||
149 | // --------------------------------------------------------------------------- | ||
150 | |||
151 | private async createTranscodingJobsWithChildren (options: { | ||
152 | videoUUID: string | ||
153 | parent: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload) | ||
154 | children: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)[] | ||
155 | user: MUserId | null | ||
156 | }) { | ||
157 | const { videoUUID, parent, children, user } = options | ||
158 | |||
159 | const parentJob = await this.buildTranscodingJob({ payload: parent, user }) | ||
160 | const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) | ||
161 | |||
162 | await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) | ||
163 | |||
164 | await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) | ||
165 | } | ||
166 | |||
167 | private async buildTranscodingJob (options: { | ||
168 | payload: VideoTranscodingPayload | ||
169 | user: MUserId | null // null means we don't want priority | ||
170 | }) { | ||
171 | const { user, payload } = options | ||
172 | |||
173 | return { | ||
174 | type: 'video-transcoding' as 'video-transcoding', | ||
175 | priority: await this.getTranscodingJobPriority({ user, fallback: undefined }), | ||
176 | payload | ||
177 | } | ||
178 | } | ||
179 | |||
180 | private async buildLowerResolutionJobPayloads (options: { | ||
181 | video: MVideoWithFileThumbnail | ||
182 | inputVideoResolution: number | ||
183 | inputVideoFPS: number | ||
184 | hasAudio: boolean | ||
185 | isNewVideo: boolean | ||
186 | }) { | ||
187 | const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options | ||
188 | |||
189 | // Create transcoding jobs if there are enabled resolutions | ||
190 | const resolutionsEnabled = await Hooks.wrapObject( | ||
191 | computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), | ||
192 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
193 | options | ||
194 | ) | ||
195 | |||
196 | const sequentialPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | ||
197 | |||
198 | for (const resolution of resolutionsEnabled) { | ||
199 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | ||
200 | |||
201 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { | ||
202 | const payloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ | ||
203 | this.buildWebTorrentJobPayload({ | ||
204 | videoUUID: video.uuid, | ||
205 | resolution, | ||
206 | fps, | ||
207 | isNewVideo | ||
208 | }) | ||
209 | ] | ||
210 | |||
211 | // Create a subsequent job to create HLS resolution that will just copy web video codecs | ||
212 | if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
213 | payloads.push( | ||
214 | this.buildHLSJobPayload({ | ||
215 | videoUUID: video.uuid, | ||
216 | resolution, | ||
217 | fps, | ||
218 | isNewVideo, | ||
219 | copyCodecs: true | ||
220 | }) | ||
221 | ) | ||
222 | } | ||
223 | |||
224 | sequentialPayloads.push(payloads) | ||
225 | } else if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
226 | sequentialPayloads.push([ | ||
227 | this.buildHLSJobPayload({ | ||
228 | videoUUID: video.uuid, | ||
229 | resolution, | ||
230 | fps, | ||
231 | copyCodecs: false, | ||
232 | isNewVideo | ||
233 | }) | ||
234 | ]) | ||
235 | } | ||
236 | } | ||
237 | |||
238 | return sequentialPayloads | ||
239 | } | ||
240 | |||
241 | private buildHLSJobPayload (options: { | ||
242 | videoUUID: string | ||
243 | resolution: number | ||
244 | fps: number | ||
245 | isNewVideo: boolean | ||
246 | deleteWebTorrentFiles?: boolean // default false | ||
247 | copyCodecs?: boolean // default false | ||
248 | }): HLSTranscodingPayload { | ||
249 | const { videoUUID, resolution, fps, isNewVideo, deleteWebTorrentFiles = false, copyCodecs = false } = options | ||
250 | |||
251 | return { | ||
252 | type: 'new-resolution-to-hls', | ||
253 | videoUUID, | ||
254 | resolution, | ||
255 | fps, | ||
256 | copyCodecs, | ||
257 | isNewVideo, | ||
258 | deleteWebTorrentFiles | ||
259 | } | ||
260 | } | ||
261 | |||
262 | private buildWebTorrentJobPayload (options: { | ||
263 | videoUUID: string | ||
264 | resolution: number | ||
265 | fps: number | ||
266 | isNewVideo: boolean | ||
267 | }): NewWebTorrentResolutionTranscodingPayload { | ||
268 | const { videoUUID, resolution, fps, isNewVideo } = options | ||
269 | |||
270 | return { | ||
271 | type: 'new-resolution-to-webtorrent', | ||
272 | videoUUID, | ||
273 | isNewVideo, | ||
274 | resolution, | ||
275 | fps | ||
276 | } | ||
277 | } | ||
278 | |||
279 | private buildMergeAudioPayload (options: { | ||
280 | videoUUID: string | ||
281 | isNewVideo: boolean | ||
282 | }): MergeAudioTranscodingPayload { | ||
283 | const { videoUUID, isNewVideo } = options | ||
284 | |||
285 | return { | ||
286 | type: 'merge-audio-to-webtorrent', | ||
287 | resolution: DEFAULT_AUDIO_RESOLUTION, | ||
288 | fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, | ||
289 | videoUUID, | ||
290 | isNewVideo | ||
291 | } | ||
292 | } | ||
293 | |||
294 | private buildOptimizePayload (options: { | ||
295 | videoUUID: string | ||
296 | quickTranscode: boolean | ||
297 | isNewVideo: boolean | ||
298 | }): OptimizeTranscodingPayload { | ||
299 | const { videoUUID, quickTranscode, isNewVideo } = options | ||
300 | |||
301 | return { | ||
302 | type: 'optimize-to-webtorrent', | ||
303 | videoUUID, | ||
304 | isNewVideo, | ||
305 | quickTranscode | ||
306 | } | ||
307 | } | ||
308 | } | ||
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts new file mode 100644 index 000000000..c7a63d2e2 --- /dev/null +++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts | |||
@@ -0,0 +1,189 @@ | |||
1 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
5 | import { Hooks } from '@server/lib/plugins/hooks' | ||
6 | import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners' | ||
7 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
8 | import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' | ||
9 | import { MRunnerJob } from '@server/types/models/runners' | ||
10 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' | ||
11 | import { computeResolutionsToTranscode } from '../../transcoding-resolutions' | ||
12 | import { AbstractJobBuilder } from './abstract-job-builder' | ||
13 | |||
14 | /** | ||
15 | * | ||
16 | * Class to build transcoding job in the local job queue | ||
17 | * | ||
18 | */ | ||
19 | |||
20 | const lTags = loggerTagsFactory('transcoding') | ||
21 | |||
22 | export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | ||
23 | |||
24 | async createOptimizeOrMergeAudioJobs (options: { | ||
25 | video: MVideoFullLight | ||
26 | videoFile: MVideoFile | ||
27 | isNewVideo: boolean | ||
28 | user: MUserId | ||
29 | }) { | ||
30 | const { video, videoFile, isNewVideo, user } = options | ||
31 | |||
32 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
33 | |||
34 | try { | ||
35 | await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { | ||
36 | const probe = await ffprobePromise(videoFilePath) | ||
37 | |||
38 | const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) | ||
39 | const hasAudio = await hasAudioStream(videoFilePath, probe) | ||
40 | const inputFPS = videoFile.isAudio() | ||
41 | ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value | ||
42 | : await getVideoStreamFPS(videoFilePath, probe) | ||
43 | |||
44 | const maxResolution = await isAudioFile(videoFilePath, probe) | ||
45 | ? DEFAULT_AUDIO_RESOLUTION | ||
46 | : resolution | ||
47 | |||
48 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
49 | const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
50 | |||
51 | const mainRunnerJob = videoFile.isAudio() | ||
52 | ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) | ||
53 | : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) | ||
54 | |||
55 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | ||
56 | await new VODHLSTranscodingJobHandler().create({ | ||
57 | video, | ||
58 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false, | ||
59 | resolution: maxResolution, | ||
60 | fps, | ||
61 | isNewVideo, | ||
62 | dependsOnRunnerJob: mainRunnerJob, | ||
63 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
64 | }) | ||
65 | } | ||
66 | |||
67 | await this.buildLowerResolutionJobPayloads({ | ||
68 | video, | ||
69 | inputVideoResolution: maxResolution, | ||
70 | inputVideoFPS: inputFPS, | ||
71 | hasAudio, | ||
72 | isNewVideo, | ||
73 | mainRunnerJob, | ||
74 | user | ||
75 | }) | ||
76 | }) | ||
77 | } finally { | ||
78 | mutexReleaser() | ||
79 | } | ||
80 | } | ||
81 | |||
82 | // --------------------------------------------------------------------------- | ||
83 | |||
84 | async createTranscodingJobs (options: { | ||
85 | transcodingType: 'hls' | 'webtorrent' | ||
86 | video: MVideoFullLight | ||
87 | resolutions: number[] | ||
88 | isNewVideo: boolean | ||
89 | user: MUserId | null | ||
90 | }) { | ||
91 | const { video, transcodingType, resolutions, isNewVideo, user } = options | ||
92 | |||
93 | const maxResolution = Math.max(...resolutions) | ||
94 | const { fps: inputFPS } = await video.probeMaxQualityFile() | ||
95 | const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
96 | const priority = await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
97 | |||
98 | const childrenResolutions = resolutions.filter(r => r !== maxResolution) | ||
99 | |||
100 | logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) | ||
101 | |||
102 | // Process the last resolution before the other ones to prevent concurrency issue | ||
103 | // Because low resolutions use the biggest one as ffmpeg input | ||
104 | const mainJob = transcodingType === 'hls' | ||
105 | // eslint-disable-next-line max-len | ||
106 | ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) | ||
107 | : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) | ||
108 | |||
109 | for (const resolution of childrenResolutions) { | ||
110 | const dependsOnRunnerJob = mainJob | ||
111 | const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) | ||
112 | |||
113 | if (transcodingType === 'hls') { | ||
114 | await new VODHLSTranscodingJobHandler().create({ | ||
115 | video, | ||
116 | resolution, | ||
117 | fps, | ||
118 | isNewVideo, | ||
119 | deleteWebVideoFiles: false, | ||
120 | dependsOnRunnerJob, | ||
121 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
122 | }) | ||
123 | continue | ||
124 | } | ||
125 | |||
126 | if (transcodingType === 'webtorrent') { | ||
127 | await new VODWebVideoTranscodingJobHandler().create({ | ||
128 | video, | ||
129 | resolution, | ||
130 | fps, | ||
131 | isNewVideo, | ||
132 | dependsOnRunnerJob, | ||
133 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
134 | }) | ||
135 | continue | ||
136 | } | ||
137 | |||
138 | throw new Error('Unknown transcoding type') | ||
139 | } | ||
140 | } | ||
141 | |||
142 | private async buildLowerResolutionJobPayloads (options: { | ||
143 | mainRunnerJob: MRunnerJob | ||
144 | video: MVideoWithFileThumbnail | ||
145 | inputVideoResolution: number | ||
146 | inputVideoFPS: number | ||
147 | hasAudio: boolean | ||
148 | isNewVideo: boolean | ||
149 | user: MUserId | ||
150 | }) { | ||
151 | const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options | ||
152 | |||
153 | // Create transcoding jobs if there are enabled resolutions | ||
154 | const resolutionsEnabled = await Hooks.wrapObject( | ||
155 | computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), | ||
156 | 'filter:transcoding.auto.resolutions-to-transcode.result', | ||
157 | options | ||
158 | ) | ||
159 | |||
160 | logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) | ||
161 | |||
162 | for (const resolution of resolutionsEnabled) { | ||
163 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | ||
164 | |||
165 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { | ||
166 | await new VODWebVideoTranscodingJobHandler().create({ | ||
167 | video, | ||
168 | resolution, | ||
169 | fps, | ||
170 | isNewVideo, | ||
171 | dependsOnRunnerJob: mainRunnerJob, | ||
172 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
173 | }) | ||
174 | } | ||
175 | |||
176 | if (CONFIG.TRANSCODING.HLS.ENABLED) { | ||
177 | await new VODHLSTranscodingJobHandler().create({ | ||
178 | video, | ||
179 | resolution, | ||
180 | fps, | ||
181 | isNewVideo, | ||
182 | deleteWebVideoFiles: false, | ||
183 | dependsOnRunnerJob: mainRunnerJob, | ||
184 | priority: await this.getTranscodingJobPriority({ user, fallback: 0 }) | ||
185 | }) | ||
186 | } | ||
187 | } | ||
188 | } | ||
189 | } | ||
diff --git a/server/lib/transcoding/transcoding-quick-transcode.ts b/server/lib/transcoding/transcoding-quick-transcode.ts new file mode 100644 index 000000000..b7f921890 --- /dev/null +++ b/server/lib/transcoding/transcoding-quick-transcode.ts | |||
@@ -0,0 +1,61 @@ | |||
1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
4 | import { getMaxBitrate } from '@shared/core-utils' | ||
5 | import { | ||
6 | ffprobePromise, | ||
7 | getAudioStream, | ||
8 | getMaxAudioBitrate, | ||
9 | getVideoStream, | ||
10 | getVideoStreamBitrate, | ||
11 | getVideoStreamDimensionsInfo, | ||
12 | getVideoStreamFPS | ||
13 | } from '@shared/ffmpeg' | ||
14 | |||
15 | export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise<boolean> { | ||
16 | if (CONFIG.TRANSCODING.PROFILE !== 'default') return false | ||
17 | |||
18 | const probe = existingProbe || await ffprobePromise(path) | ||
19 | |||
20 | return await canDoQuickVideoTranscode(path, probe) && | ||
21 | await canDoQuickAudioTranscode(path, probe) | ||
22 | } | ||
23 | |||
24 | export async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise<boolean> { | ||
25 | const parsedAudio = await getAudioStream(path, probe) | ||
26 | |||
27 | if (!parsedAudio.audioStream) return true | ||
28 | |||
29 | if (parsedAudio.audioStream['codec_name'] !== 'aac') return false | ||
30 | |||
31 | const audioBitrate = parsedAudio.bitrate | ||
32 | if (!audioBitrate) return false | ||
33 | |||
34 | const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) | ||
35 | if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false | ||
36 | |||
37 | const channelLayout = parsedAudio.audioStream['channel_layout'] | ||
38 | // Causes playback issues with Chrome | ||
39 | if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false | ||
40 | |||
41 | return true | ||
42 | } | ||
43 | |||
44 | export async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise<boolean> { | ||
45 | const videoStream = await getVideoStream(path, probe) | ||
46 | const fps = await getVideoStreamFPS(path, probe) | ||
47 | const bitRate = await getVideoStreamBitrate(path, probe) | ||
48 | const resolutionData = await getVideoStreamDimensionsInfo(path, probe) | ||
49 | |||
50 | // If ffprobe did not manage to guess the bitrate | ||
51 | if (!bitRate) return false | ||
52 | |||
53 | // check video params | ||
54 | if (!videoStream) return false | ||
55 | if (videoStream['codec_name'] !== 'h264') return false | ||
56 | if (videoStream['pix_fmt'] !== 'yuv420p') return false | ||
57 | if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false | ||
58 | if (bitRate > getMaxBitrate({ ...resolutionData, fps })) return false | ||
59 | |||
60 | return true | ||
61 | } | ||
diff --git a/server/lib/transcoding/transcoding-resolutions.ts b/server/lib/transcoding/transcoding-resolutions.ts new file mode 100644 index 000000000..91f4d18d8 --- /dev/null +++ b/server/lib/transcoding/transcoding-resolutions.ts | |||
@@ -0,0 +1,52 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { toEven } from '@shared/core-utils' | ||
3 | import { VideoResolution } from '@shared/models' | ||
4 | |||
5 | export function computeResolutionsToTranscode (options: { | ||
6 | input: number | ||
7 | type: 'vod' | 'live' | ||
8 | includeInput: boolean | ||
9 | strictLower: boolean | ||
10 | hasAudio: boolean | ||
11 | }) { | ||
12 | const { input, type, includeInput, strictLower, hasAudio } = options | ||
13 | |||
14 | const configResolutions = type === 'vod' | ||
15 | ? CONFIG.TRANSCODING.RESOLUTIONS | ||
16 | : CONFIG.LIVE.TRANSCODING.RESOLUTIONS | ||
17 | |||
18 | const resolutionsEnabled = new Set<number>() | ||
19 | |||
20 | // Put in the order we want to proceed jobs | ||
21 | const availableResolutions: VideoResolution[] = [ | ||
22 | VideoResolution.H_NOVIDEO, | ||
23 | VideoResolution.H_480P, | ||
24 | VideoResolution.H_360P, | ||
25 | VideoResolution.H_720P, | ||
26 | VideoResolution.H_240P, | ||
27 | VideoResolution.H_144P, | ||
28 | VideoResolution.H_1080P, | ||
29 | VideoResolution.H_1440P, | ||
30 | VideoResolution.H_4K | ||
31 | ] | ||
32 | |||
33 | for (const resolution of availableResolutions) { | ||
34 | // Resolution not enabled | ||
35 | if (configResolutions[resolution + 'p'] !== true) continue | ||
36 | // Too big resolution for input file | ||
37 | if (input < resolution) continue | ||
38 | // We only want lower resolutions than input file | ||
39 | if (strictLower && input === resolution) continue | ||
40 | // Audio resolutio but no audio in the video | ||
41 | if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue | ||
42 | |||
43 | resolutionsEnabled.add(resolution) | ||
44 | } | ||
45 | |||
46 | if (includeInput) { | ||
47 | // Always use an even resolution to avoid issues with ffmpeg | ||
48 | resolutionsEnabled.add(toEven(input)) | ||
49 | } | ||
50 | |||
51 | return Array.from(resolutionsEnabled) | ||
52 | } | ||
diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts deleted file mode 100644 index c7b61e9ba..000000000 --- a/server/lib/transcoding/transcoding.ts +++ /dev/null | |||
@@ -1,465 +0,0 @@ | |||
1 | import { MutexInterface } from 'async-mutex' | ||
2 | import { Job } from 'bullmq' | ||
3 | import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' | ||
4 | import { basename, extname as extnameUtil, join } from 'path' | ||
5 | import { toEven } from '@server/helpers/core-utils' | ||
6 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
7 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
8 | import { sequelizeTypescript } from '@server/initializers/database' | ||
9 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | ||
10 | import { pick } from '@shared/core-utils' | ||
11 | import { VideoResolution, VideoStorage } from '../../../shared/models/videos' | ||
12 | import { | ||
13 | buildFileMetadata, | ||
14 | canDoQuickTranscode, | ||
15 | computeResolutionsToTranscode, | ||
16 | ffprobePromise, | ||
17 | getVideoStreamDuration, | ||
18 | getVideoStreamFPS, | ||
19 | transcodeVOD, | ||
20 | TranscodeVODOptions, | ||
21 | TranscodeVODOptionsType | ||
22 | } from '../../helpers/ffmpeg' | ||
23 | import { CONFIG } from '../../initializers/config' | ||
24 | import { VideoFileModel } from '../../models/video/video-file' | ||
25 | import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' | ||
26 | import { updatePlaylistAfterFileChange } from '../hls' | ||
27 | import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' | ||
28 | import { VideoPathManager } from '../video-path-manager' | ||
29 | import { VideoTranscodingProfilesManager } from './default-transcoding-profiles' | ||
30 | |||
31 | /** | ||
32 | * | ||
33 | * Functions that run transcoding functions, update the database, cleanup files, create torrent files... | ||
34 | * Mainly called by the job queue | ||
35 | * | ||
36 | */ | ||
37 | |||
38 | // Optimize the original video file and replace it. The resolution is not changed. | ||
39 | async function optimizeOriginalVideofile (options: { | ||
40 | video: MVideoFullLight | ||
41 | inputVideoFile: MVideoFile | ||
42 | job: Job | ||
43 | }) { | ||
44 | const { video, inputVideoFile, job } = options | ||
45 | |||
46 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
47 | const newExtname = '.mp4' | ||
48 | |||
49 | // Will be released by our transcodeVOD function once ffmpeg is ran | ||
50 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
51 | |||
52 | try { | ||
53 | await video.reload() | ||
54 | |||
55 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | ||
56 | |||
57 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { | ||
58 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | ||
59 | |||
60 | const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath) | ||
61 | ? 'quick-transcode' | ||
62 | : 'video' | ||
63 | |||
64 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) | ||
65 | |||
66 | const transcodeOptions: TranscodeVODOptions = { | ||
67 | type: transcodeType, | ||
68 | |||
69 | inputPath: videoInputPath, | ||
70 | outputPath: videoTranscodedPath, | ||
71 | |||
72 | inputFileMutexReleaser, | ||
73 | |||
74 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
75 | profile: CONFIG.TRANSCODING.PROFILE, | ||
76 | |||
77 | resolution, | ||
78 | |||
79 | job | ||
80 | } | ||
81 | |||
82 | // Could be very long! | ||
83 | await transcodeVOD(transcodeOptions) | ||
84 | |||
85 | // Important to do this before getVideoFilename() to take in account the new filename | ||
86 | inputVideoFile.resolution = resolution | ||
87 | inputVideoFile.extname = newExtname | ||
88 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | ||
89 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
90 | |||
91 | const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | ||
92 | await remove(videoInputPath) | ||
93 | |||
94 | return { transcodeType, videoFile } | ||
95 | }) | ||
96 | |||
97 | return result | ||
98 | } finally { | ||
99 | inputFileMutexReleaser() | ||
100 | } | ||
101 | } | ||
102 | |||
103 | // Transcode the original video file to a lower resolution compatible with WebTorrent | ||
104 | async function transcodeNewWebTorrentResolution (options: { | ||
105 | video: MVideoFullLight | ||
106 | resolution: VideoResolution | ||
107 | job: Job | ||
108 | }) { | ||
109 | const { video, resolution, job } = options | ||
110 | |||
111 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
112 | const newExtname = '.mp4' | ||
113 | |||
114 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
115 | |||
116 | try { | ||
117 | await video.reload() | ||
118 | |||
119 | const file = video.getMaxQualityFile().withVideoOrPlaylist(video) | ||
120 | |||
121 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { | ||
122 | const newVideoFile = new VideoFileModel({ | ||
123 | resolution, | ||
124 | extname: newExtname, | ||
125 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | ||
126 | size: 0, | ||
127 | videoId: video.id | ||
128 | }) | ||
129 | |||
130 | const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) | ||
131 | |||
132 | const transcodeOptions = resolution === VideoResolution.H_NOVIDEO | ||
133 | ? { | ||
134 | type: 'only-audio' as 'only-audio', | ||
135 | |||
136 | inputPath: videoInputPath, | ||
137 | outputPath: videoTranscodedPath, | ||
138 | |||
139 | inputFileMutexReleaser, | ||
140 | |||
141 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
142 | profile: CONFIG.TRANSCODING.PROFILE, | ||
143 | |||
144 | resolution, | ||
145 | |||
146 | job | ||
147 | } | ||
148 | : { | ||
149 | type: 'video' as 'video', | ||
150 | inputPath: videoInputPath, | ||
151 | outputPath: videoTranscodedPath, | ||
152 | |||
153 | inputFileMutexReleaser, | ||
154 | |||
155 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
156 | profile: CONFIG.TRANSCODING.PROFILE, | ||
157 | |||
158 | resolution, | ||
159 | |||
160 | job | ||
161 | } | ||
162 | |||
163 | await transcodeVOD(transcodeOptions) | ||
164 | |||
165 | return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile) | ||
166 | }) | ||
167 | |||
168 | return result | ||
169 | } finally { | ||
170 | inputFileMutexReleaser() | ||
171 | } | ||
172 | } | ||
173 | |||
174 | // Merge an image with an audio file to create a video | ||
175 | async function mergeAudioVideofile (options: { | ||
176 | video: MVideoFullLight | ||
177 | resolution: VideoResolution | ||
178 | job: Job | ||
179 | }) { | ||
180 | const { video, resolution, job } = options | ||
181 | |||
182 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
183 | const newExtname = '.mp4' | ||
184 | |||
185 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
186 | |||
187 | try { | ||
188 | await video.reload() | ||
189 | |||
190 | const inputVideoFile = video.getMinQualityFile() | ||
191 | |||
192 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | ||
193 | |||
194 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { | ||
195 | const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | ||
196 | |||
197 | // If the user updates the video preview during transcoding | ||
198 | const previewPath = video.getPreview().getPath() | ||
199 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | ||
200 | await copyFile(previewPath, tmpPreviewPath) | ||
201 | |||
202 | const transcodeOptions = { | ||
203 | type: 'merge-audio' as 'merge-audio', | ||
204 | |||
205 | inputPath: tmpPreviewPath, | ||
206 | outputPath: videoTranscodedPath, | ||
207 | |||
208 | inputFileMutexReleaser, | ||
209 | |||
210 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
211 | profile: CONFIG.TRANSCODING.PROFILE, | ||
212 | |||
213 | audioPath: audioInputPath, | ||
214 | resolution, | ||
215 | |||
216 | job | ||
217 | } | ||
218 | |||
219 | try { | ||
220 | await transcodeVOD(transcodeOptions) | ||
221 | |||
222 | await remove(audioInputPath) | ||
223 | await remove(tmpPreviewPath) | ||
224 | } catch (err) { | ||
225 | await remove(tmpPreviewPath) | ||
226 | throw err | ||
227 | } | ||
228 | |||
229 | // Important to do this before getVideoFilename() to take in account the new file extension | ||
230 | inputVideoFile.extname = newExtname | ||
231 | inputVideoFile.resolution = resolution | ||
232 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | ||
233 | |||
234 | // ffmpeg generated a new video file, so update the video duration | ||
235 | // See https://trac.ffmpeg.org/ticket/5456 | ||
236 | video.duration = await getVideoStreamDuration(videoTranscodedPath) | ||
237 | await video.save() | ||
238 | |||
239 | return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile) | ||
240 | }) | ||
241 | |||
242 | return result | ||
243 | } finally { | ||
244 | inputFileMutexReleaser() | ||
245 | } | ||
246 | } | ||
247 | |||
248 | // Concat TS segments from a live video to a fragmented mp4 HLS playlist | ||
249 | async function generateHlsPlaylistResolutionFromTS (options: { | ||
250 | video: MVideo | ||
251 | concatenatedTsFilePath: string | ||
252 | resolution: VideoResolution | ||
253 | isAAC: boolean | ||
254 | inputFileMutexReleaser: MutexInterface.Releaser | ||
255 | }) { | ||
256 | return generateHlsPlaylistCommon({ | ||
257 | type: 'hls-from-ts' as 'hls-from-ts', | ||
258 | inputPath: options.concatenatedTsFilePath, | ||
259 | |||
260 | ...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ]) | ||
261 | }) | ||
262 | } | ||
263 | |||
264 | // Generate an HLS playlist from an input file, and update the master playlist | ||
265 | function generateHlsPlaylistResolution (options: { | ||
266 | video: MVideo | ||
267 | videoInputPath: string | ||
268 | resolution: VideoResolution | ||
269 | copyCodecs: boolean | ||
270 | inputFileMutexReleaser: MutexInterface.Releaser | ||
271 | job?: Job | ||
272 | }) { | ||
273 | return generateHlsPlaylistCommon({ | ||
274 | type: 'hls' as 'hls', | ||
275 | inputPath: options.videoInputPath, | ||
276 | |||
277 | ...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) | ||
278 | }) | ||
279 | } | ||
280 | |||
281 | // --------------------------------------------------------------------------- | ||
282 | |||
283 | export { | ||
284 | generateHlsPlaylistResolution, | ||
285 | generateHlsPlaylistResolutionFromTS, | ||
286 | optimizeOriginalVideofile, | ||
287 | transcodeNewWebTorrentResolution, | ||
288 | mergeAudioVideofile | ||
289 | } | ||
290 | |||
291 | // --------------------------------------------------------------------------- | ||
292 | |||
293 | async function onWebTorrentVideoFileTranscoding ( | ||
294 | video: MVideoFullLight, | ||
295 | videoFile: MVideoFile, | ||
296 | transcodingPath: string, | ||
297 | newVideoFile: MVideoFile | ||
298 | ) { | ||
299 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
300 | |||
301 | try { | ||
302 | await video.reload() | ||
303 | |||
304 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) | ||
305 | |||
306 | const stats = await stat(transcodingPath) | ||
307 | |||
308 | const probe = await ffprobePromise(transcodingPath) | ||
309 | const fps = await getVideoStreamFPS(transcodingPath, probe) | ||
310 | const metadata = await buildFileMetadata(transcodingPath, probe) | ||
311 | |||
312 | await move(transcodingPath, outputPath, { overwrite: true }) | ||
313 | |||
314 | videoFile.size = stats.size | ||
315 | videoFile.fps = fps | ||
316 | videoFile.metadata = metadata | ||
317 | |||
318 | await createTorrentAndSetInfoHash(video, videoFile) | ||
319 | |||
320 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | ||
321 | if (oldFile) await video.removeWebTorrentFile(oldFile) | ||
322 | |||
323 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | ||
324 | video.VideoFiles = await video.$get('VideoFiles') | ||
325 | |||
326 | return { video, videoFile } | ||
327 | } finally { | ||
328 | mutexReleaser() | ||
329 | } | ||
330 | } | ||
331 | |||
332 | async function generateHlsPlaylistCommon (options: { | ||
333 | type: 'hls' | 'hls-from-ts' | ||
334 | video: MVideo | ||
335 | inputPath: string | ||
336 | resolution: VideoResolution | ||
337 | |||
338 | inputFileMutexReleaser: MutexInterface.Releaser | ||
339 | |||
340 | copyCodecs?: boolean | ||
341 | isAAC?: boolean | ||
342 | |||
343 | job?: Job | ||
344 | }) { | ||
345 | const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options | ||
346 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
347 | |||
348 | const videoTranscodedBasePath = join(transcodeDirectory, type) | ||
349 | await ensureDir(videoTranscodedBasePath) | ||
350 | |||
351 | const videoFilename = generateHLSVideoFilename(resolution) | ||
352 | const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) | ||
353 | const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename) | ||
354 | |||
355 | const transcodeOptions = { | ||
356 | type, | ||
357 | |||
358 | inputPath, | ||
359 | outputPath: resolutionPlaylistFileTranscodePath, | ||
360 | |||
361 | availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), | ||
362 | profile: CONFIG.TRANSCODING.PROFILE, | ||
363 | |||
364 | resolution, | ||
365 | copyCodecs, | ||
366 | |||
367 | isAAC, | ||
368 | |||
369 | inputFileMutexReleaser, | ||
370 | |||
371 | hlsPlaylist: { | ||
372 | videoFilename | ||
373 | }, | ||
374 | |||
375 | job | ||
376 | } | ||
377 | |||
378 | await transcodeVOD(transcodeOptions) | ||
379 | |||
380 | // Create or update the playlist | ||
381 | const playlist = await retryTransactionWrapper(() => { | ||
382 | return sequelizeTypescript.transaction(async transaction => { | ||
383 | return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) | ||
384 | }) | ||
385 | }) | ||
386 | |||
387 | const newVideoFile = new VideoFileModel({ | ||
388 | resolution, | ||
389 | extname: extnameUtil(videoFilename), | ||
390 | size: 0, | ||
391 | filename: videoFilename, | ||
392 | fps: -1, | ||
393 | videoStreamingPlaylistId: playlist.id | ||
394 | }) | ||
395 | |||
396 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
397 | |||
398 | try { | ||
399 | // VOD transcoding is a long task, refresh video attributes | ||
400 | await video.reload() | ||
401 | |||
402 | const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile) | ||
403 | await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) | ||
404 | |||
405 | // Move playlist file | ||
406 | const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename) | ||
407 | await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) | ||
408 | // Move video file | ||
409 | await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) | ||
410 | |||
411 | // Update video duration if it was not set (in case of a live for example) | ||
412 | if (!video.duration) { | ||
413 | video.duration = await getVideoStreamDuration(videoFilePath) | ||
414 | await video.save() | ||
415 | } | ||
416 | |||
417 | const stats = await stat(videoFilePath) | ||
418 | |||
419 | newVideoFile.size = stats.size | ||
420 | newVideoFile.fps = await getVideoStreamFPS(videoFilePath) | ||
421 | newVideoFile.metadata = await buildFileMetadata(videoFilePath) | ||
422 | |||
423 | await createTorrentAndSetInfoHash(playlist, newVideoFile) | ||
424 | |||
425 | const oldFile = await VideoFileModel.loadHLSFile({ | ||
426 | playlistId: playlist.id, | ||
427 | fps: newVideoFile.fps, | ||
428 | resolution: newVideoFile.resolution | ||
429 | }) | ||
430 | |||
431 | if (oldFile) { | ||
432 | await video.removeStreamingPlaylistVideoFile(playlist, oldFile) | ||
433 | await oldFile.destroy() | ||
434 | } | ||
435 | |||
436 | const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) | ||
437 | |||
438 | await updatePlaylistAfterFileChange(video, playlist) | ||
439 | |||
440 | return { resolutionPlaylistPath, videoFile: savedVideoFile } | ||
441 | } finally { | ||
442 | mutexReleaser() | ||
443 | } | ||
444 | } | ||
445 | |||
446 | function buildOriginalFileResolution (inputResolution: number) { | ||
447 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { | ||
448 | return toEven(inputResolution) | ||
449 | } | ||
450 | |||
451 | const resolutions = computeResolutionsToTranscode({ | ||
452 | input: inputResolution, | ||
453 | type: 'vod', | ||
454 | includeInput: false, | ||
455 | strictLower: false, | ||
456 | // We don't really care about the audio resolution in this context | ||
457 | hasAudio: true | ||
458 | }) | ||
459 | |||
460 | if (resolutions.length === 0) { | ||
461 | return toEven(inputResolution) | ||
462 | } | ||
463 | |||
464 | return Math.max(...resolutions) | ||
465 | } | ||
diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts new file mode 100644 index 000000000..d43d03b2a --- /dev/null +++ b/server/lib/transcoding/web-transcoding.ts | |||
@@ -0,0 +1,273 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import { copyFile, move, remove, stat } from 'fs-extra' | ||
3 | import { basename, join } from 'path' | ||
4 | import { computeOutputFPS } from '@server/helpers/ffmpeg' | ||
5 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | ||
6 | import { MVideoFile, MVideoFullLight } from '@server/types/models' | ||
7 | import { toEven } from '@shared/core-utils' | ||
8 | import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg' | ||
9 | import { VideoResolution, VideoStorage } from '@shared/models' | ||
10 | import { CONFIG } from '../../initializers/config' | ||
11 | import { VideoFileModel } from '../../models/video/video-file' | ||
12 | import { generateWebTorrentVideoFilename } from '../paths' | ||
13 | import { buildFileMetadata } from '../video-file' | ||
14 | import { VideoPathManager } from '../video-path-manager' | ||
15 | import { buildFFmpegVOD } from './shared' | ||
16 | import { computeResolutionsToTranscode } from './transcoding-resolutions' | ||
17 | |||
18 | // Optimize the original video file and replace it. The resolution is not changed. | ||
19 | export async function optimizeOriginalVideofile (options: { | ||
20 | video: MVideoFullLight | ||
21 | inputVideoFile: MVideoFile | ||
22 | quickTranscode: boolean | ||
23 | job: Job | ||
24 | }) { | ||
25 | const { video, inputVideoFile, quickTranscode, job } = options | ||
26 | |||
27 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
28 | const newExtname = '.mp4' | ||
29 | |||
30 | // Will be released by our transcodeVOD function once ffmpeg is ran | ||
31 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
32 | |||
33 | try { | ||
34 | await video.reload() | ||
35 | |||
36 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | ||
37 | |||
38 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { | ||
39 | const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | ||
40 | |||
41 | const transcodeType: TranscodeVODOptionsType = quickTranscode | ||
42 | ? 'quick-transcode' | ||
43 | : 'video' | ||
44 | |||
45 | const resolution = buildOriginalFileResolution(inputVideoFile.resolution) | ||
46 | const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution }) | ||
47 | |||
48 | // Could be very long! | ||
49 | await buildFFmpegVOD(job).transcode({ | ||
50 | type: transcodeType, | ||
51 | |||
52 | inputPath: videoInputPath, | ||
53 | outputPath: videoOutputPath, | ||
54 | |||
55 | inputFileMutexReleaser, | ||
56 | |||
57 | resolution, | ||
58 | fps | ||
59 | }) | ||
60 | |||
61 | // Important to do this before getVideoFilename() to take in account the new filename | ||
62 | inputVideoFile.resolution = resolution | ||
63 | inputVideoFile.extname = newExtname | ||
64 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | ||
65 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | ||
66 | |||
67 | const { videoFile } = await onWebTorrentVideoFileTranscoding({ | ||
68 | video, | ||
69 | videoFile: inputVideoFile, | ||
70 | videoOutputPath | ||
71 | }) | ||
72 | |||
73 | await remove(videoInputPath) | ||
74 | |||
75 | return { transcodeType, videoFile } | ||
76 | }) | ||
77 | |||
78 | return result | ||
79 | } finally { | ||
80 | inputFileMutexReleaser() | ||
81 | } | ||
82 | } | ||
83 | |||
84 | // Transcode the original video file to a lower resolution compatible with WebTorrent | ||
85 | export async function transcodeNewWebTorrentResolution (options: { | ||
86 | video: MVideoFullLight | ||
87 | resolution: VideoResolution | ||
88 | fps: number | ||
89 | job: Job | ||
90 | }) { | ||
91 | const { video, resolution, fps, job } = options | ||
92 | |||
93 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
94 | const newExtname = '.mp4' | ||
95 | |||
96 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
97 | |||
98 | try { | ||
99 | await video.reload() | ||
100 | |||
101 | const file = video.getMaxQualityFile().withVideoOrPlaylist(video) | ||
102 | |||
103 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { | ||
104 | const newVideoFile = new VideoFileModel({ | ||
105 | resolution, | ||
106 | extname: newExtname, | ||
107 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | ||
108 | size: 0, | ||
109 | videoId: video.id | ||
110 | }) | ||
111 | |||
112 | const videoOutputPath = join(transcodeDirectory, newVideoFile.filename) | ||
113 | |||
114 | const transcodeOptions = { | ||
115 | type: 'video' as 'video', | ||
116 | |||
117 | inputPath: videoInputPath, | ||
118 | outputPath: videoOutputPath, | ||
119 | |||
120 | inputFileMutexReleaser, | ||
121 | |||
122 | resolution, | ||
123 | fps | ||
124 | } | ||
125 | |||
126 | await buildFFmpegVOD(job).transcode(transcodeOptions) | ||
127 | |||
128 | return onWebTorrentVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) | ||
129 | }) | ||
130 | |||
131 | return result | ||
132 | } finally { | ||
133 | inputFileMutexReleaser() | ||
134 | } | ||
135 | } | ||
136 | |||
137 | // Merge an image with an audio file to create a video | ||
138 | export async function mergeAudioVideofile (options: { | ||
139 | video: MVideoFullLight | ||
140 | resolution: VideoResolution | ||
141 | fps: number | ||
142 | job: Job | ||
143 | }) { | ||
144 | const { video, resolution, fps, job } = options | ||
145 | |||
146 | const transcodeDirectory = CONFIG.STORAGE.TMP_DIR | ||
147 | const newExtname = '.mp4' | ||
148 | |||
149 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
150 | |||
151 | try { | ||
152 | await video.reload() | ||
153 | |||
154 | const inputVideoFile = video.getMinQualityFile() | ||
155 | |||
156 | const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) | ||
157 | |||
158 | const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { | ||
159 | const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) | ||
160 | |||
161 | // If the user updates the video preview during transcoding | ||
162 | const previewPath = video.getPreview().getPath() | ||
163 | const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) | ||
164 | await copyFile(previewPath, tmpPreviewPath) | ||
165 | |||
166 | const transcodeOptions = { | ||
167 | type: 'merge-audio' as 'merge-audio', | ||
168 | |||
169 | inputPath: tmpPreviewPath, | ||
170 | outputPath: videoOutputPath, | ||
171 | |||
172 | inputFileMutexReleaser, | ||
173 | |||
174 | audioPath: audioInputPath, | ||
175 | resolution, | ||
176 | fps | ||
177 | } | ||
178 | |||
179 | try { | ||
180 | await buildFFmpegVOD(job).transcode(transcodeOptions) | ||
181 | |||
182 | await remove(audioInputPath) | ||
183 | await remove(tmpPreviewPath) | ||
184 | } catch (err) { | ||
185 | await remove(tmpPreviewPath) | ||
186 | throw err | ||
187 | } | ||
188 | |||
189 | // Important to do this before getVideoFilename() to take in account the new file extension | ||
190 | inputVideoFile.extname = newExtname | ||
191 | inputVideoFile.resolution = resolution | ||
192 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | ||
193 | |||
194 | // ffmpeg generated a new video file, so update the video duration | ||
195 | // See https://trac.ffmpeg.org/ticket/5456 | ||
196 | video.duration = await getVideoStreamDuration(videoOutputPath) | ||
197 | await video.save() | ||
198 | |||
199 | return onWebTorrentVideoFileTranscoding({ | ||
200 | video, | ||
201 | videoFile: inputVideoFile, | ||
202 | videoOutputPath | ||
203 | }) | ||
204 | }) | ||
205 | |||
206 | return result | ||
207 | } finally { | ||
208 | inputFileMutexReleaser() | ||
209 | } | ||
210 | } | ||
211 | |||
212 | export async function onWebTorrentVideoFileTranscoding (options: { | ||
213 | video: MVideoFullLight | ||
214 | videoFile: MVideoFile | ||
215 | videoOutputPath: string | ||
216 | }) { | ||
217 | const { video, videoFile, videoOutputPath } = options | ||
218 | |||
219 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | ||
220 | |||
221 | try { | ||
222 | await video.reload() | ||
223 | |||
224 | const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) | ||
225 | |||
226 | const stats = await stat(videoOutputPath) | ||
227 | |||
228 | const probe = await ffprobePromise(videoOutputPath) | ||
229 | const fps = await getVideoStreamFPS(videoOutputPath, probe) | ||
230 | const metadata = await buildFileMetadata(videoOutputPath, probe) | ||
231 | |||
232 | await move(videoOutputPath, outputPath, { overwrite: true }) | ||
233 | |||
234 | videoFile.size = stats.size | ||
235 | videoFile.fps = fps | ||
236 | videoFile.metadata = metadata | ||
237 | |||
238 | await createTorrentAndSetInfoHash(video, videoFile) | ||
239 | |||
240 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | ||
241 | if (oldFile) await video.removeWebTorrentFile(oldFile) | ||
242 | |||
243 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | ||
244 | video.VideoFiles = await video.$get('VideoFiles') | ||
245 | |||
246 | return { video, videoFile } | ||
247 | } finally { | ||
248 | mutexReleaser() | ||
249 | } | ||
250 | } | ||
251 | |||
252 | // --------------------------------------------------------------------------- | ||
253 | |||
254 | function buildOriginalFileResolution (inputResolution: number) { | ||
255 | if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { | ||
256 | return toEven(inputResolution) | ||
257 | } | ||
258 | |||
259 | const resolutions = computeResolutionsToTranscode({ | ||
260 | input: inputResolution, | ||
261 | type: 'vod', | ||
262 | includeInput: false, | ||
263 | strictLower: false, | ||
264 | // We don't really care about the audio resolution in this context | ||
265 | hasAudio: true | ||
266 | }) | ||
267 | |||
268 | if (resolutions.length === 0) { | ||
269 | return toEven(inputResolution) | ||
270 | } | ||
271 | |||
272 | return Math.max(...resolutions) | ||
273 | } | ||
diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts index 58040cb6d..c7e0eb414 100644 --- a/server/lib/uploadx.ts +++ b/server/lib/uploadx.ts | |||
@@ -3,6 +3,7 @@ import { buildLogger } from '@server/helpers/logger' | |||
3 | import { getResumableUploadPath } from '@server/helpers/upload' | 3 | import { getResumableUploadPath } from '@server/helpers/upload' |
4 | import { CONFIG } from '@server/initializers/config' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { LogLevel, Uploadx } from '@uploadx/core' | 5 | import { LogLevel, Uploadx } from '@uploadx/core' |
6 | import { extname } from 'path' | ||
6 | 7 | ||
7 | const logger = buildLogger('uploadx') | 8 | const logger = buildLogger('uploadx') |
8 | 9 | ||
@@ -26,7 +27,9 @@ const uploadx = new Uploadx({ | |||
26 | if (!res.locals.oauth) return undefined | 27 | if (!res.locals.oauth) return undefined |
27 | 28 | ||
28 | return res.locals.oauth.token.user.id + '' | 29 | return res.locals.oauth.token.user.id + '' |
29 | } | 30 | }, |
31 | |||
32 | filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}` | ||
30 | }) | 33 | }) |
31 | 34 | ||
32 | export { | 35 | export { |
diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts index fd5837a3a..cb1ea834c 100644 --- a/server/lib/video-blacklist.ts +++ b/server/lib/video-blacklist.ts | |||
@@ -81,7 +81,7 @@ async function blacklistVideo (videoInstance: MVideoAccountLight, options: Video | |||
81 | } | 81 | } |
82 | 82 | ||
83 | if (videoInstance.isLive) { | 83 | if (videoInstance.isLive) { |
84 | LiveManager.Instance.stopSessionOf(videoInstance.id, LiveVideoError.BLACKLISTED) | 84 | LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED) |
85 | } | 85 | } |
86 | 86 | ||
87 | Notifier.Instance.notifyOnVideoBlacklist(blacklist) | 87 | Notifier.Instance.notifyOnVideoBlacklist(blacklist) |
diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts index 2ab7190f1..8fcc3c253 100644 --- a/server/lib/video-file.ts +++ b/server/lib/video-file.ts | |||
@@ -1,6 +1,44 @@ | |||
1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
1 | import { logger } from '@server/helpers/logger' | 2 | import { logger } from '@server/helpers/logger' |
3 | import { VideoFileModel } from '@server/models/video/video-file' | ||
2 | import { MVideoWithAllFiles } from '@server/types/models' | 4 | import { MVideoWithAllFiles } from '@server/types/models' |
5 | import { getLowercaseExtension } from '@shared/core-utils' | ||
6 | import { getFileSize } from '@shared/extra-utils' | ||
7 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' | ||
8 | import { VideoFileMetadata, VideoResolution } from '@shared/models' | ||
3 | import { lTags } from './object-storage/shared' | 9 | import { lTags } from './object-storage/shared' |
10 | import { generateHLSVideoFilename, generateWebTorrentVideoFilename } from './paths' | ||
11 | |||
12 | async function buildNewFile (options: { | ||
13 | path: string | ||
14 | mode: 'web-video' | 'hls' | ||
15 | }) { | ||
16 | const { path, mode } = options | ||
17 | |||
18 | const probe = await ffprobePromise(path) | ||
19 | const size = await getFileSize(path) | ||
20 | |||
21 | const videoFile = new VideoFileModel({ | ||
22 | extname: getLowercaseExtension(path), | ||
23 | size, | ||
24 | metadata: await buildFileMetadata(path, probe) | ||
25 | }) | ||
26 | |||
27 | if (await isAudioFile(path, probe)) { | ||
28 | videoFile.resolution = VideoResolution.H_NOVIDEO | ||
29 | } else { | ||
30 | videoFile.fps = await getVideoStreamFPS(path, probe) | ||
31 | videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution | ||
32 | } | ||
33 | |||
34 | videoFile.filename = mode === 'web-video' | ||
35 | ? generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) | ||
36 | : generateHLSVideoFilename(videoFile.resolution) | ||
37 | |||
38 | return videoFile | ||
39 | } | ||
40 | |||
41 | // --------------------------------------------------------------------------- | ||
4 | 42 | ||
5 | async function removeHLSPlaylist (video: MVideoWithAllFiles) { | 43 | async function removeHLSPlaylist (video: MVideoWithAllFiles) { |
6 | const hls = video.getHLSPlaylist() | 44 | const hls = video.getHLSPlaylist() |
@@ -61,9 +99,23 @@ async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId: | |||
61 | return video | 99 | return video |
62 | } | 100 | } |
63 | 101 | ||
102 | // --------------------------------------------------------------------------- | ||
103 | |||
104 | async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { | ||
105 | const metadata = existingProbe || await ffprobePromise(path) | ||
106 | |||
107 | return new VideoFileMetadata(metadata) | ||
108 | } | ||
109 | |||
110 | // --------------------------------------------------------------------------- | ||
111 | |||
64 | export { | 112 | export { |
113 | buildNewFile, | ||
114 | |||
65 | removeHLSPlaylist, | 115 | removeHLSPlaylist, |
66 | removeHLSFile, | 116 | removeHLSFile, |
67 | removeAllWebTorrentFiles, | 117 | removeAllWebTorrentFiles, |
68 | removeWebTorrentFile | 118 | removeWebTorrentFile, |
119 | |||
120 | buildFileMetadata | ||
69 | } | 121 | } |
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index cdacd35f2..b392bdb00 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { MVideoFullLight } from '@server/types/models' | 1 | import { MVideoFullLight } from '@server/types/models' |
2 | import { getVideoStreamDuration } from '@shared/extra-utils' | 2 | import { getVideoStreamDuration } from '@shared/ffmpeg' |
3 | import { VideoStudioTask } from '@shared/models' | 3 | import { VideoStudioTask } from '@shared/models' |
4 | 4 | ||
5 | function buildTaskFileFieldname (indice: number, fieldName = 'file') { | 5 | function buildTaskFileFieldname (indice: number, fieldName = 'file') { |
diff --git a/server/lib/video.ts b/server/lib/video.ts index aacc41a7a..588dc553f 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -2,14 +2,14 @@ import { UploadFiles } from 'express' | |||
2 | import memoizee from 'memoizee' | 2 | import memoizee from 'memoizee' |
3 | import { Transaction } from 'sequelize/types' | 3 | import { Transaction } from 'sequelize/types' |
4 | import { CONFIG } from '@server/initializers/config' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' | 5 | import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' |
6 | import { TagModel } from '@server/models/video/tag' | 6 | import { TagModel } from '@server/models/video/tag' |
7 | import { VideoModel } from '@server/models/video/video' | 7 | import { VideoModel } from '@server/models/video/video' |
8 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 8 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
9 | import { FilteredModelAttributes } from '@server/types' | 9 | import { FilteredModelAttributes } from '@server/types' |
10 | import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | 10 | import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' |
11 | import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models' | 11 | import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' |
12 | import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue' | 12 | import { CreateJobArgument, JobQueue } from './job-queue/job-queue' |
13 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 13 | import { updateVideoMiniatureFromExisting } from './thumbnail' |
14 | import { moveFilesIfPrivacyChanged } from './video-privacy' | 14 | import { moveFilesIfPrivacyChanged } from './video-privacy' |
15 | 15 | ||
@@ -87,58 +87,6 @@ async function setVideoTags (options: { | |||
87 | 87 | ||
88 | // --------------------------------------------------------------------------- | 88 | // --------------------------------------------------------------------------- |
89 | 89 | ||
90 | async function buildOptimizeOrMergeAudioJob (options: { | ||
91 | video: MVideoUUID | ||
92 | videoFile: MVideoFile | ||
93 | user: MUserId | ||
94 | isNewVideo?: boolean // Default true | ||
95 | }) { | ||
96 | const { video, videoFile, user, isNewVideo } = options | ||
97 | |||
98 | let payload: VideoTranscodingPayload | ||
99 | |||
100 | if (videoFile.isAudio()) { | ||
101 | payload = { | ||
102 | type: 'merge-audio-to-webtorrent', | ||
103 | resolution: DEFAULT_AUDIO_RESOLUTION, | ||
104 | videoUUID: video.uuid, | ||
105 | createHLSIfNeeded: true, | ||
106 | isNewVideo | ||
107 | } | ||
108 | } else { | ||
109 | payload = { | ||
110 | type: 'optimize-to-webtorrent', | ||
111 | videoUUID: video.uuid, | ||
112 | isNewVideo | ||
113 | } | ||
114 | } | ||
115 | |||
116 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
117 | |||
118 | return { | ||
119 | type: 'video-transcoding' as 'video-transcoding', | ||
120 | priority: await getTranscodingJobPriority(user), | ||
121 | payload | ||
122 | } | ||
123 | } | ||
124 | |||
125 | async function buildTranscodingJob (payload: VideoTranscodingPayload, options: CreateJobOptions = {}) { | ||
126 | await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') | ||
127 | |||
128 | return { type: 'video-transcoding' as 'video-transcoding', payload, ...options } | ||
129 | } | ||
130 | |||
131 | async function getTranscodingJobPriority (user: MUserId) { | ||
132 | const now = new Date() | ||
133 | const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) | ||
134 | |||
135 | const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) | ||
136 | |||
137 | return JOB_PRIORITY.TRANSCODING + videoUploadedByUser | ||
138 | } | ||
139 | |||
140 | // --------------------------------------------------------------------------- | ||
141 | |||
142 | async function buildMoveToObjectStorageJob (options: { | 90 | async function buildMoveToObjectStorageJob (options: { |
143 | video: MVideoUUID | 91 | video: MVideoUUID |
144 | previousVideoState: VideoState | 92 | previousVideoState: VideoState |
@@ -235,10 +183,7 @@ export { | |||
235 | buildLocalVideoFromReq, | 183 | buildLocalVideoFromReq, |
236 | buildVideoThumbnailsFromReq, | 184 | buildVideoThumbnailsFromReq, |
237 | setVideoTags, | 185 | setVideoTags, |
238 | buildOptimizeOrMergeAudioJob, | ||
239 | buildTranscodingJob, | ||
240 | buildMoveToObjectStorageJob, | 186 | buildMoveToObjectStorageJob, |
241 | getTranscodingJobPriority, | ||
242 | addVideoJobsAfterUpdate, | 187 | addVideoJobsAfterUpdate, |
243 | getCachedVideoDuration | 188 | getCachedVideoDuration |
244 | } | 189 | } |
diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts index e6025c8ce..0eefa2a8e 100644 --- a/server/middlewares/auth.ts +++ b/server/middlewares/auth.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { Socket } from 'socket.io' | 2 | import { Socket } from 'socket.io' |
3 | import { getAccessToken } from '@server/lib/auth/oauth-model' | 3 | import { getAccessToken } from '@server/lib/auth/oauth-model' |
4 | import { RunnerModel } from '@server/models/runner/runner' | ||
4 | import { HttpStatusCode } from '../../shared/models/http/http-error-codes' | 5 | import { HttpStatusCode } from '../../shared/models/http/http-error-codes' |
5 | import { logger } from '../helpers/logger' | 6 | import { logger } from '../helpers/logger' |
6 | import { handleOAuthAuthenticate } from '../lib/auth/oauth' | 7 | import { handleOAuthAuthenticate } from '../lib/auth/oauth' |
@@ -27,7 +28,7 @@ function authenticate (req: express.Request, res: express.Response, next: expres | |||
27 | function authenticateSocket (socket: Socket, next: (err?: any) => void) { | 28 | function authenticateSocket (socket: Socket, next: (err?: any) => void) { |
28 | const accessToken = socket.handshake.query['accessToken'] | 29 | const accessToken = socket.handshake.query['accessToken'] |
29 | 30 | ||
30 | logger.debug('Checking socket access token %s.', accessToken) | 31 | logger.debug('Checking access token in runner.') |
31 | 32 | ||
32 | if (!accessToken) return next(new Error('No access token provided')) | 33 | if (!accessToken) return next(new Error('No access token provided')) |
33 | if (typeof accessToken !== 'string') return next(new Error('Access token is invalid')) | 34 | if (typeof accessToken !== 'string') return next(new Error('Access token is invalid')) |
@@ -73,9 +74,31 @@ function optionalAuthenticate (req: express.Request, res: express.Response, next | |||
73 | 74 | ||
74 | // --------------------------------------------------------------------------- | 75 | // --------------------------------------------------------------------------- |
75 | 76 | ||
77 | function authenticateRunnerSocket (socket: Socket, next: (err?: any) => void) { | ||
78 | const runnerToken = socket.handshake.auth['runnerToken'] | ||
79 | |||
80 | logger.debug('Checking runner token in socket.') | ||
81 | |||
82 | if (!runnerToken) return next(new Error('No runner token provided')) | ||
83 | if (typeof runnerToken !== 'string') return next(new Error('Runner token is invalid')) | ||
84 | |||
85 | RunnerModel.loadByToken(runnerToken) | ||
86 | .then(runner => { | ||
87 | if (!runner) return next(new Error('Invalid runner token.')) | ||
88 | |||
89 | socket.handshake.auth.runner = runner | ||
90 | |||
91 | return next() | ||
92 | }) | ||
93 | .catch(err => logger.error('Cannot get runner token.', { err })) | ||
94 | } | ||
95 | |||
96 | // --------------------------------------------------------------------------- | ||
97 | |||
76 | export { | 98 | export { |
77 | authenticate, | 99 | authenticate, |
78 | authenticateSocket, | 100 | authenticateSocket, |
79 | authenticatePromise, | 101 | authenticatePromise, |
80 | optionalAuthenticate | 102 | optionalAuthenticate, |
103 | authenticateRunnerSocket | ||
81 | } | 104 | } |
diff --git a/server/middlewares/doc.ts b/server/middlewares/doc.ts index c43f41977..eef76acaa 100644 --- a/server/middlewares/doc.ts +++ b/server/middlewares/doc.ts | |||
@@ -5,7 +5,7 @@ function openapiOperationDoc (options: { | |||
5 | operationId?: string | 5 | operationId?: string |
6 | }) { | 6 | }) { |
7 | return (req: express.Request, res: express.Response, next: express.NextFunction) => { | 7 | return (req: express.Request, res: express.Response, next: express.NextFunction) => { |
8 | res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api/rest-reference.html#operation/' + options.operationId | 8 | res.locals.docUrl = options.url || 'https://docs.joinpeertube.org/api-rest-reference.html#operation/' + options.operationId |
9 | 9 | ||
10 | if (next) return next() | 10 | if (next) return next() |
11 | } | 11 | } |
diff --git a/server/middlewares/error.ts b/server/middlewares/error.ts index 540edaeeb..94762e355 100644 --- a/server/middlewares/error.ts +++ b/server/middlewares/error.ts | |||
@@ -5,7 +5,7 @@ import { HttpStatusCode } from '@shared/models' | |||
5 | 5 | ||
6 | function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) { | 6 | function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) { |
7 | res.fail = options => { | 7 | res.fail = options => { |
8 | const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance } = options | 8 | const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance, tags } = options |
9 | 9 | ||
10 | const extension = new ProblemDocumentExtension({ | 10 | const extension = new ProblemDocumentExtension({ |
11 | ...data, | 11 | ...data, |
@@ -31,11 +31,11 @@ function apiFailMiddleware (req: express.Request, res: express.Response, next: e | |||
31 | detail: message, | 31 | detail: message, |
32 | 32 | ||
33 | type: type | 33 | type: type |
34 | ? `https://docs.joinpeertube.org/api/rest-reference.html#section/Errors/${type}` | 34 | ? `https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/${type}` |
35 | : undefined | 35 | : undefined |
36 | }, extension) | 36 | }, extension) |
37 | 37 | ||
38 | logger.debug('Bad HTTP request.', { json }) | 38 | logger.debug('Bad HTTP request.', { json, tags }) |
39 | 39 | ||
40 | res.json(json) | 40 | res.json(json) |
41 | } | 41 | } |
diff --git a/server/middlewares/rate-limiter.ts b/server/middlewares/rate-limiter.ts index bc9513969..1eef8b360 100644 --- a/server/middlewares/rate-limiter.ts +++ b/server/middlewares/rate-limiter.ts | |||
@@ -1,10 +1,12 @@ | |||
1 | import express from 'express' | ||
2 | import RateLimit, { Options as RateLimitHandlerOptions } from 'express-rate-limit' | ||
3 | import { RunnerModel } from '@server/models/runner/runner' | ||
1 | import { UserRole } from '@shared/models' | 4 | import { UserRole } from '@shared/models' |
2 | import RateLimit from 'express-rate-limit' | ||
3 | import { optionalAuthenticate } from './auth' | 5 | import { optionalAuthenticate } from './auth' |
4 | 6 | ||
5 | const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ]) | 7 | const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ]) |
6 | 8 | ||
7 | function buildRateLimiter (options: { | 9 | export function buildRateLimiter (options: { |
8 | windowMs: number | 10 | windowMs: number |
9 | max: number | 11 | max: number |
10 | skipFailedRequests?: boolean | 12 | skipFailedRequests?: boolean |
@@ -15,17 +17,33 @@ function buildRateLimiter (options: { | |||
15 | skipFailedRequests: options.skipFailedRequests, | 17 | skipFailedRequests: options.skipFailedRequests, |
16 | 18 | ||
17 | handler: (req, res, next, options) => { | 19 | handler: (req, res, next, options) => { |
20 | // Bypass rate limit for registered runners | ||
21 | if (req.body?.runnerToken) { | ||
22 | return RunnerModel.loadByToken(req.body.runnerToken) | ||
23 | .then(runner => { | ||
24 | if (runner) return next() | ||
25 | |||
26 | return sendRateLimited(res, options) | ||
27 | }) | ||
28 | } | ||
29 | |||
30 | // Bypass rate limit for admins/moderators | ||
18 | return optionalAuthenticate(req, res, () => { | 31 | return optionalAuthenticate(req, res, () => { |
19 | if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) { | 32 | if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) { |
20 | return next() | 33 | return next() |
21 | } | 34 | } |
22 | 35 | ||
23 | return res.status(options.statusCode).send(options.message) | 36 | return sendRateLimited(res, options) |
24 | }) | 37 | }) |
25 | } | 38 | } |
26 | }) | 39 | }) |
27 | } | 40 | } |
28 | 41 | ||
29 | export { | 42 | // --------------------------------------------------------------------------- |
30 | buildRateLimiter | 43 | // Private |
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | function sendRateLimited (res: express.Response, options: RateLimitHandlerOptions) { | ||
47 | return res.status(options.statusCode).send(options.message) | ||
48 | |||
31 | } | 49 | } |
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 4a9d1cb54..b3e7e5011 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts | |||
@@ -54,6 +54,7 @@ const customConfigUpdateValidator = [ | |||
54 | body('transcoding.resolutions.1080p').isBoolean(), | 54 | body('transcoding.resolutions.1080p').isBoolean(), |
55 | body('transcoding.resolutions.1440p').isBoolean(), | 55 | body('transcoding.resolutions.1440p').isBoolean(), |
56 | body('transcoding.resolutions.2160p').isBoolean(), | 56 | body('transcoding.resolutions.2160p').isBoolean(), |
57 | body('transcoding.remoteRunners.enabled').isBoolean(), | ||
57 | 58 | ||
58 | body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(), | 59 | body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(), |
59 | 60 | ||
@@ -97,6 +98,7 @@ const customConfigUpdateValidator = [ | |||
97 | body('live.transcoding.resolutions.1440p').isBoolean(), | 98 | body('live.transcoding.resolutions.1440p').isBoolean(), |
98 | body('live.transcoding.resolutions.2160p').isBoolean(), | 99 | body('live.transcoding.resolutions.2160p').isBoolean(), |
99 | body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(), | 100 | body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(), |
101 | body('live.transcoding.remoteRunners.enabled').isBoolean(), | ||
100 | 102 | ||
101 | body('search.remoteUri.users').isBoolean(), | 103 | body('search.remoteUri.users').isBoolean(), |
102 | body('search.remoteUri.anonymous').isBoolean(), | 104 | body('search.remoteUri.anonymous').isBoolean(), |
diff --git a/server/middlewares/validators/runners/index.ts b/server/middlewares/validators/runners/index.ts new file mode 100644 index 000000000..9a9629a80 --- /dev/null +++ b/server/middlewares/validators/runners/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './jobs' | ||
2 | export * from './registration-token' | ||
3 | export * from './runners' | ||
diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts new file mode 100644 index 000000000..56afa39aa --- /dev/null +++ b/server/middlewares/validators/runners/job-files.ts | |||
@@ -0,0 +1,27 @@ | |||
1 | import express from 'express' | ||
2 | import { HttpStatusCode } from '@shared/models' | ||
3 | import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' | ||
4 | |||
5 | const tags = [ 'runner' ] | ||
6 | |||
7 | export const runnerJobGetVideoTranscodingFileValidator = [ | ||
8 | isValidVideoIdParam('videoId'), | ||
9 | |||
10 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
11 | if (areValidationErrors(req, res)) return | ||
12 | |||
13 | if (!await doesVideoExist(req.params.videoId, res, 'all')) return | ||
14 | |||
15 | const runnerJob = res.locals.runnerJob | ||
16 | |||
17 | if (runnerJob.privatePayload.videoUUID !== res.locals.videoAll.uuid) { | ||
18 | return res.fail({ | ||
19 | status: HttpStatusCode.FORBIDDEN_403, | ||
20 | message: 'Job is not associated to this video', | ||
21 | tags: [ ...tags, res.locals.videoAll.uuid ] | ||
22 | }) | ||
23 | } | ||
24 | |||
25 | return next() | ||
26 | } | ||
27 | ] | ||
diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts new file mode 100644 index 000000000..8cb87e946 --- /dev/null +++ b/server/middlewares/validators/runners/jobs.ts | |||
@@ -0,0 +1,156 @@ | |||
1 | import express from 'express' | ||
2 | import { body, param } from 'express-validator' | ||
3 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' | ||
4 | import { | ||
5 | isRunnerJobAbortReasonValid, | ||
6 | isRunnerJobErrorMessageValid, | ||
7 | isRunnerJobProgressValid, | ||
8 | isRunnerJobSuccessPayloadValid, | ||
9 | isRunnerJobTokenValid, | ||
10 | isRunnerJobUpdatePayloadValid | ||
11 | } from '@server/helpers/custom-validators/runners/jobs' | ||
12 | import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners' | ||
13 | import { cleanUpReqFiles } from '@server/helpers/express-utils' | ||
14 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
15 | import { HttpStatusCode, RunnerJobState, RunnerJobSuccessBody, RunnerJobUpdateBody, ServerErrorCode } from '@shared/models' | ||
16 | import { areValidationErrors } from '../shared' | ||
17 | |||
18 | const tags = [ 'runner' ] | ||
19 | |||
20 | export const acceptRunnerJobValidator = [ | ||
21 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
22 | if (res.locals.runnerJob.state !== RunnerJobState.PENDING) { | ||
23 | return res.fail({ | ||
24 | status: HttpStatusCode.BAD_REQUEST_400, | ||
25 | message: 'This runner job is not in pending state', | ||
26 | tags | ||
27 | }) | ||
28 | } | ||
29 | |||
30 | return next() | ||
31 | } | ||
32 | ] | ||
33 | |||
34 | export const abortRunnerJobValidator = [ | ||
35 | body('reason').custom(isRunnerJobAbortReasonValid), | ||
36 | |||
37 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
38 | if (areValidationErrors(req, res, { tags })) return | ||
39 | |||
40 | return next() | ||
41 | } | ||
42 | ] | ||
43 | |||
44 | export const updateRunnerJobValidator = [ | ||
45 | body('progress').optional().custom(isRunnerJobProgressValid), | ||
46 | |||
47 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
48 | if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) | ||
49 | |||
50 | const body = req.body as RunnerJobUpdateBody | ||
51 | |||
52 | if (isRunnerJobUpdatePayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) { | ||
53 | cleanUpReqFiles(req) | ||
54 | |||
55 | return res.fail({ | ||
56 | status: HttpStatusCode.BAD_REQUEST_400, | ||
57 | message: 'Payload is invalid', | ||
58 | tags | ||
59 | }) | ||
60 | } | ||
61 | |||
62 | return next() | ||
63 | } | ||
64 | ] | ||
65 | |||
66 | export const errorRunnerJobValidator = [ | ||
67 | body('message').custom(isRunnerJobErrorMessageValid), | ||
68 | |||
69 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
70 | if (areValidationErrors(req, res, { tags })) return | ||
71 | |||
72 | return next() | ||
73 | } | ||
74 | ] | ||
75 | |||
76 | export const successRunnerJobValidator = [ | ||
77 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
78 | const body = req.body as RunnerJobSuccessBody | ||
79 | |||
80 | if (isRunnerJobSuccessPayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) { | ||
81 | cleanUpReqFiles(req) | ||
82 | |||
83 | return res.fail({ | ||
84 | status: HttpStatusCode.BAD_REQUEST_400, | ||
85 | message: 'Payload is invalid', | ||
86 | tags | ||
87 | }) | ||
88 | } | ||
89 | |||
90 | return next() | ||
91 | } | ||
92 | ] | ||
93 | |||
94 | export const runnerJobGetValidator = [ | ||
95 | param('jobUUID').custom(isUUIDValid), | ||
96 | |||
97 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
98 | if (areValidationErrors(req, res, { tags })) return | ||
99 | |||
100 | const runnerJob = await RunnerJobModel.loadWithRunner(req.params.jobUUID) | ||
101 | |||
102 | if (!runnerJob) { | ||
103 | return res.fail({ | ||
104 | status: HttpStatusCode.NOT_FOUND_404, | ||
105 | message: 'Unknown runner job', | ||
106 | tags | ||
107 | }) | ||
108 | } | ||
109 | |||
110 | res.locals.runnerJob = runnerJob | ||
111 | |||
112 | return next() | ||
113 | } | ||
114 | ] | ||
115 | |||
116 | export const jobOfRunnerGetValidator = [ | ||
117 | param('jobUUID').custom(isUUIDValid), | ||
118 | |||
119 | body('runnerToken').custom(isRunnerTokenValid), | ||
120 | body('jobToken').custom(isRunnerJobTokenValid), | ||
121 | |||
122 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
123 | if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) | ||
124 | |||
125 | const runnerJob = await RunnerJobModel.loadByRunnerAndJobTokensWithRunner({ | ||
126 | uuid: req.params.jobUUID, | ||
127 | runnerToken: req.body.runnerToken, | ||
128 | jobToken: req.body.jobToken | ||
129 | }) | ||
130 | |||
131 | if (!runnerJob) { | ||
132 | cleanUpReqFiles(req) | ||
133 | |||
134 | return res.fail({ | ||
135 | status: HttpStatusCode.NOT_FOUND_404, | ||
136 | message: 'Unknown runner job', | ||
137 | tags | ||
138 | }) | ||
139 | } | ||
140 | |||
141 | if (runnerJob.state !== RunnerJobState.PROCESSING) { | ||
142 | cleanUpReqFiles(req) | ||
143 | |||
144 | return res.fail({ | ||
145 | status: HttpStatusCode.BAD_REQUEST_400, | ||
146 | type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE, | ||
147 | message: 'Job is not in "processing" state', | ||
148 | tags | ||
149 | }) | ||
150 | } | ||
151 | |||
152 | res.locals.runnerJob = runnerJob | ||
153 | |||
154 | return next() | ||
155 | } | ||
156 | ] | ||
diff --git a/server/middlewares/validators/runners/registration-token.ts b/server/middlewares/validators/runners/registration-token.ts new file mode 100644 index 000000000..cc31d4a7e --- /dev/null +++ b/server/middlewares/validators/runners/registration-token.ts | |||
@@ -0,0 +1,37 @@ | |||
1 | import express from 'express' | ||
2 | import { param } from 'express-validator' | ||
3 | import { isIdValid } from '@server/helpers/custom-validators/misc' | ||
4 | import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' | ||
5 | import { forceNumber } from '@shared/core-utils' | ||
6 | import { HttpStatusCode } from '@shared/models' | ||
7 | import { areValidationErrors } from '../shared/utils' | ||
8 | |||
9 | const tags = [ 'runner' ] | ||
10 | |||
11 | const deleteRegistrationTokenValidator = [ | ||
12 | param('id').custom(isIdValid), | ||
13 | |||
14 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
15 | if (areValidationErrors(req, res, { tags })) return | ||
16 | |||
17 | const registrationToken = await RunnerRegistrationTokenModel.load(forceNumber(req.params.id)) | ||
18 | |||
19 | if (!registrationToken) { | ||
20 | return res.fail({ | ||
21 | status: HttpStatusCode.NOT_FOUND_404, | ||
22 | message: 'Registration token not found', | ||
23 | tags | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | res.locals.runnerRegistrationToken = registrationToken | ||
28 | |||
29 | return next() | ||
30 | } | ||
31 | ] | ||
32 | |||
33 | // --------------------------------------------------------------------------- | ||
34 | |||
35 | export { | ||
36 | deleteRegistrationTokenValidator | ||
37 | } | ||
diff --git a/server/middlewares/validators/runners/runners.ts b/server/middlewares/validators/runners/runners.ts new file mode 100644 index 000000000..71a1275d2 --- /dev/null +++ b/server/middlewares/validators/runners/runners.ts | |||
@@ -0,0 +1,95 @@ | |||
1 | import express from 'express' | ||
2 | import { body, param } from 'express-validator' | ||
3 | import { isIdValid } from '@server/helpers/custom-validators/misc' | ||
4 | import { | ||
5 | isRunnerDescriptionValid, | ||
6 | isRunnerNameValid, | ||
7 | isRunnerRegistrationTokenValid, | ||
8 | isRunnerTokenValid | ||
9 | } from '@server/helpers/custom-validators/runners/runners' | ||
10 | import { RunnerModel } from '@server/models/runner/runner' | ||
11 | import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' | ||
12 | import { forceNumber } from '@shared/core-utils' | ||
13 | import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@shared/models' | ||
14 | import { areValidationErrors } from '../shared/utils' | ||
15 | |||
16 | const tags = [ 'runner' ] | ||
17 | |||
18 | const registerRunnerValidator = [ | ||
19 | body('registrationToken').custom(isRunnerRegistrationTokenValid), | ||
20 | body('name').custom(isRunnerNameValid), | ||
21 | body('description').optional().custom(isRunnerDescriptionValid), | ||
22 | |||
23 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
24 | if (areValidationErrors(req, res, { tags })) return | ||
25 | |||
26 | const body: RegisterRunnerBody = req.body | ||
27 | |||
28 | const runnerRegistrationToken = await RunnerRegistrationTokenModel.loadByRegistrationToken(body.registrationToken) | ||
29 | |||
30 | if (!runnerRegistrationToken) { | ||
31 | return res.fail({ | ||
32 | status: HttpStatusCode.NOT_FOUND_404, | ||
33 | message: 'Registration token is invalid', | ||
34 | tags | ||
35 | }) | ||
36 | } | ||
37 | |||
38 | res.locals.runnerRegistrationToken = runnerRegistrationToken | ||
39 | |||
40 | return next() | ||
41 | } | ||
42 | ] | ||
43 | |||
44 | const deleteRunnerValidator = [ | ||
45 | param('runnerId').custom(isIdValid), | ||
46 | |||
47 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
48 | if (areValidationErrors(req, res, { tags })) return | ||
49 | |||
50 | const runner = await RunnerModel.load(forceNumber(req.params.runnerId)) | ||
51 | |||
52 | if (!runner) { | ||
53 | return res.fail({ | ||
54 | status: HttpStatusCode.NOT_FOUND_404, | ||
55 | message: 'Runner not found', | ||
56 | tags | ||
57 | }) | ||
58 | } | ||
59 | |||
60 | res.locals.runner = runner | ||
61 | |||
62 | return next() | ||
63 | } | ||
64 | ] | ||
65 | |||
66 | const getRunnerFromTokenValidator = [ | ||
67 | body('runnerToken').custom(isRunnerTokenValid), | ||
68 | |||
69 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
70 | if (areValidationErrors(req, res, { tags })) return | ||
71 | |||
72 | const runner = await RunnerModel.loadByToken(req.body.runnerToken) | ||
73 | |||
74 | if (!runner) { | ||
75 | return res.fail({ | ||
76 | status: HttpStatusCode.NOT_FOUND_404, | ||
77 | message: 'Unknown runner token', | ||
78 | type: ServerErrorCode.UNKNOWN_RUNNER_TOKEN, | ||
79 | tags | ||
80 | }) | ||
81 | } | ||
82 | |||
83 | res.locals.runner = runner | ||
84 | |||
85 | return next() | ||
86 | } | ||
87 | ] | ||
88 | |||
89 | // --------------------------------------------------------------------------- | ||
90 | |||
91 | export { | ||
92 | registerRunnerValidator, | ||
93 | deleteRunnerValidator, | ||
94 | getRunnerFromTokenValidator | ||
95 | } | ||
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index e6cc46317..959f663ac 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts | |||
@@ -34,6 +34,10 @@ export const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COL | |||
34 | 34 | ||
35 | export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS) | 35 | export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS) |
36 | 36 | ||
37 | export const runnersSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNERS) | ||
38 | export const runnerRegistrationTokensSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_REGISTRATION_TOKENS) | ||
39 | export const runnerJobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_JOBS) | ||
40 | |||
37 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
38 | 42 | ||
39 | function checkSortFactory (columns: string[], tags: string[] = []) { | 43 | function checkSortFactory (columns: string[], tags: string[] = []) { |
diff --git a/server/middlewares/validators/videos/video-live.ts b/server/middlewares/validators/videos/video-live.ts index e80fe1593..2aff831a8 100644 --- a/server/middlewares/validators/videos/video-live.ts +++ b/server/middlewares/validators/videos/video-live.ts | |||
@@ -115,6 +115,15 @@ const videoLiveAddValidator = getCommonVideoEditAttributes().concat([ | |||
115 | }) | 115 | }) |
116 | } | 116 | } |
117 | 117 | ||
118 | if (body.saveReplay && !body.replaySettings?.privacy) { | ||
119 | cleanUpReqFiles(req) | ||
120 | |||
121 | return res.fail({ | ||
122 | status: HttpStatusCode.BAD_REQUEST_400, | ||
123 | message: 'Live replay is enabled but privacy replay setting is missing' | ||
124 | }) | ||
125 | } | ||
126 | |||
118 | const user = res.locals.oauth.token.User | 127 | const user = res.locals.oauth.token.User |
119 | if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req) | 128 | if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req) |
120 | 129 | ||
diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts index b3e2d8101..4397e887e 100644 --- a/server/middlewares/validators/videos/video-studio.ts +++ b/server/middlewares/validators/videos/video-studio.ts | |||
@@ -10,7 +10,7 @@ import { | |||
10 | import { cleanUpReqFiles } from '@server/helpers/express-utils' | 10 | import { cleanUpReqFiles } from '@server/helpers/express-utils' |
11 | import { CONFIG } from '@server/initializers/config' | 11 | import { CONFIG } from '@server/initializers/config' |
12 | import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' | 12 | import { approximateIntroOutroAdditionalSize, getTaskFile } from '@server/lib/video-studio' |
13 | import { isAudioFile } from '@shared/extra-utils' | 13 | import { isAudioFile } from '@shared/ffmpeg' |
14 | import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' | 14 | import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' |
15 | import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' | 15 | import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' |
16 | 16 | ||
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index d3014e8e7..794e1d4f1 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts | |||
@@ -7,6 +7,7 @@ import { getServerActor } from '@server/models/application/application' | |||
7 | import { ExpressPromiseHandler } from '@server/types/express-handler' | 7 | import { ExpressPromiseHandler } from '@server/types/express-handler' |
8 | import { MUserAccountId, MVideoFullLight } from '@server/types/models' | 8 | import { MUserAccountId, MVideoFullLight } from '@server/types/models' |
9 | import { arrayify, getAllPrivacies } from '@shared/core-utils' | 9 | import { arrayify, getAllPrivacies } from '@shared/core-utils' |
10 | import { getVideoStreamDuration } from '@shared/ffmpeg' | ||
10 | import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models' | 11 | import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models' |
11 | import { | 12 | import { |
12 | exists, | 13 | exists, |
@@ -37,7 +38,6 @@ import { | |||
37 | isVideoSupportValid | 38 | isVideoSupportValid |
38 | } from '../../../helpers/custom-validators/videos' | 39 | } from '../../../helpers/custom-validators/videos' |
39 | import { cleanUpReqFiles } from '../../../helpers/express-utils' | 40 | import { cleanUpReqFiles } from '../../../helpers/express-utils' |
40 | import { getVideoStreamDuration } from '../../../helpers/ffmpeg' | ||
41 | import { logger } from '../../../helpers/logger' | 41 | import { logger } from '../../../helpers/logger' |
42 | import { deleteFileAndCatch } from '../../../helpers/utils' | 42 | import { deleteFileAndCatch } from '../../../helpers/utils' |
43 | import { getVideoWithAttributes } from '../../../helpers/video' | 43 | import { getVideoWithAttributes } from '../../../helpers/video' |
diff --git a/server/models/runner/runner-job.ts b/server/models/runner/runner-job.ts new file mode 100644 index 000000000..add6f9a43 --- /dev/null +++ b/server/models/runner/runner-job.ts | |||
@@ -0,0 +1,347 @@ | |||
1 | import { FindOptions, Op, Transaction } from 'sequelize' | ||
2 | import { | ||
3 | AllowNull, | ||
4 | BelongsTo, | ||
5 | Column, | ||
6 | CreatedAt, | ||
7 | DataType, | ||
8 | Default, | ||
9 | ForeignKey, | ||
10 | IsUUID, | ||
11 | Model, | ||
12 | Scopes, | ||
13 | Table, | ||
14 | UpdatedAt | ||
15 | } from 'sequelize-typescript' | ||
16 | import { isUUIDValid } from '@server/helpers/custom-validators/misc' | ||
17 | import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants' | ||
18 | import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners' | ||
19 | import { RunnerJob, RunnerJobAdmin, RunnerJobPayload, RunnerJobPrivatePayload, RunnerJobState, RunnerJobType } from '@shared/models' | ||
20 | import { AttributesOnly } from '@shared/typescript-utils' | ||
21 | import { getSort, searchAttribute } from '../shared' | ||
22 | import { RunnerModel } from './runner' | ||
23 | |||
24 | enum ScopeNames { | ||
25 | WITH_RUNNER = 'WITH_RUNNER', | ||
26 | WITH_PARENT = 'WITH_PARENT' | ||
27 | } | ||
28 | |||
29 | @Scopes(() => ({ | ||
30 | [ScopeNames.WITH_RUNNER]: { | ||
31 | include: [ | ||
32 | { | ||
33 | model: RunnerModel.unscoped(), | ||
34 | required: false | ||
35 | } | ||
36 | ] | ||
37 | }, | ||
38 | [ScopeNames.WITH_PARENT]: { | ||
39 | include: [ | ||
40 | { | ||
41 | model: RunnerJobModel.unscoped(), | ||
42 | required: false | ||
43 | } | ||
44 | ] | ||
45 | } | ||
46 | })) | ||
47 | @Table({ | ||
48 | tableName: 'runnerJob', | ||
49 | indexes: [ | ||
50 | { | ||
51 | fields: [ 'uuid' ], | ||
52 | unique: true | ||
53 | }, | ||
54 | { | ||
55 | fields: [ 'processingJobToken' ], | ||
56 | unique: true | ||
57 | }, | ||
58 | { | ||
59 | fields: [ 'runnerId' ] | ||
60 | } | ||
61 | ] | ||
62 | }) | ||
63 | export class RunnerJobModel extends Model<Partial<AttributesOnly<RunnerJobModel>>> { | ||
64 | |||
65 | @AllowNull(false) | ||
66 | @IsUUID(4) | ||
67 | @Column(DataType.UUID) | ||
68 | uuid: string | ||
69 | |||
70 | @AllowNull(false) | ||
71 | @Column | ||
72 | type: RunnerJobType | ||
73 | |||
74 | @AllowNull(false) | ||
75 | @Column(DataType.JSONB) | ||
76 | payload: RunnerJobPayload | ||
77 | |||
78 | @AllowNull(false) | ||
79 | @Column(DataType.JSONB) | ||
80 | privatePayload: RunnerJobPrivatePayload | ||
81 | |||
82 | @AllowNull(false) | ||
83 | @Column | ||
84 | state: RunnerJobState | ||
85 | |||
86 | @AllowNull(false) | ||
87 | @Default(0) | ||
88 | @Column | ||
89 | failures: number | ||
90 | |||
91 | @AllowNull(true) | ||
92 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNER_JOBS.ERROR_MESSAGE.max)) | ||
93 | error: string | ||
94 | |||
95 | // Less has priority | ||
96 | @AllowNull(false) | ||
97 | @Column | ||
98 | priority: number | ||
99 | |||
100 | // Used to fetch the appropriate job when the runner wants to post the result | ||
101 | @AllowNull(true) | ||
102 | @Column | ||
103 | processingJobToken: string | ||
104 | |||
105 | @AllowNull(true) | ||
106 | @Column | ||
107 | progress: number | ||
108 | |||
109 | @AllowNull(true) | ||
110 | @Column | ||
111 | startedAt: Date | ||
112 | |||
113 | @AllowNull(true) | ||
114 | @Column | ||
115 | finishedAt: Date | ||
116 | |||
117 | @CreatedAt | ||
118 | createdAt: Date | ||
119 | |||
120 | @UpdatedAt | ||
121 | updatedAt: Date | ||
122 | |||
123 | @ForeignKey(() => RunnerJobModel) | ||
124 | @Column | ||
125 | dependsOnRunnerJobId: number | ||
126 | |||
127 | @BelongsTo(() => RunnerJobModel, { | ||
128 | foreignKey: { | ||
129 | name: 'dependsOnRunnerJobId', | ||
130 | allowNull: true | ||
131 | }, | ||
132 | onDelete: 'cascade' | ||
133 | }) | ||
134 | DependsOnRunnerJob: RunnerJobModel | ||
135 | |||
136 | @ForeignKey(() => RunnerModel) | ||
137 | @Column | ||
138 | runnerId: number | ||
139 | |||
140 | @BelongsTo(() => RunnerModel, { | ||
141 | foreignKey: { | ||
142 | name: 'runnerId', | ||
143 | allowNull: true | ||
144 | }, | ||
145 | onDelete: 'SET NULL' | ||
146 | }) | ||
147 | Runner: RunnerModel | ||
148 | |||
149 | // --------------------------------------------------------------------------- | ||
150 | |||
151 | static loadWithRunner (uuid: string) { | ||
152 | const query = { | ||
153 | where: { uuid } | ||
154 | } | ||
155 | |||
156 | return RunnerJobModel.scope(ScopeNames.WITH_RUNNER).findOne<MRunnerJobRunner>(query) | ||
157 | } | ||
158 | |||
159 | static loadByRunnerAndJobTokensWithRunner (options: { | ||
160 | uuid: string | ||
161 | runnerToken: string | ||
162 | jobToken: string | ||
163 | }) { | ||
164 | const { uuid, runnerToken, jobToken } = options | ||
165 | |||
166 | const query = { | ||
167 | where: { | ||
168 | uuid, | ||
169 | processingJobToken: jobToken | ||
170 | }, | ||
171 | include: { | ||
172 | model: RunnerModel.unscoped(), | ||
173 | required: true, | ||
174 | where: { | ||
175 | runnerToken | ||
176 | } | ||
177 | } | ||
178 | } | ||
179 | |||
180 | return RunnerJobModel.findOne<MRunnerJobRunner>(query) | ||
181 | } | ||
182 | |||
183 | static listAvailableJobs () { | ||
184 | const query = { | ||
185 | limit: 10, | ||
186 | order: getSort('priority'), | ||
187 | where: { | ||
188 | state: RunnerJobState.PENDING | ||
189 | } | ||
190 | } | ||
191 | |||
192 | return RunnerJobModel.findAll<MRunnerJob>(query) | ||
193 | } | ||
194 | |||
195 | static listStalledJobs (options: { | ||
196 | staleTimeMS: number | ||
197 | types: RunnerJobType[] | ||
198 | }) { | ||
199 | const before = new Date(Date.now() - options.staleTimeMS) | ||
200 | |||
201 | return RunnerJobModel.findAll<MRunnerJob>({ | ||
202 | where: { | ||
203 | type: { | ||
204 | [Op.in]: options.types | ||
205 | }, | ||
206 | state: RunnerJobState.PROCESSING, | ||
207 | updatedAt: { | ||
208 | [Op.lt]: before | ||
209 | } | ||
210 | } | ||
211 | }) | ||
212 | } | ||
213 | |||
214 | static listChildrenOf (job: MRunnerJob, transaction?: Transaction) { | ||
215 | const query = { | ||
216 | where: { | ||
217 | dependsOnRunnerJobId: job.id | ||
218 | }, | ||
219 | transaction | ||
220 | } | ||
221 | |||
222 | return RunnerJobModel.findAll<MRunnerJob>(query) | ||
223 | } | ||
224 | |||
225 | static listForApi (options: { | ||
226 | start: number | ||
227 | count: number | ||
228 | sort: string | ||
229 | search?: string | ||
230 | }) { | ||
231 | const { start, count, sort, search } = options | ||
232 | |||
233 | const query: FindOptions = { | ||
234 | offset: start, | ||
235 | limit: count, | ||
236 | order: getSort(sort) | ||
237 | } | ||
238 | |||
239 | if (search) { | ||
240 | if (isUUIDValid(search)) { | ||
241 | query.where = { uuid: search } | ||
242 | } else { | ||
243 | query.where = { | ||
244 | [Op.or]: [ | ||
245 | searchAttribute(search, 'type'), | ||
246 | searchAttribute(search, '$Runner.name$') | ||
247 | ] | ||
248 | } | ||
249 | } | ||
250 | } | ||
251 | |||
252 | return Promise.all([ | ||
253 | RunnerJobModel.scope([ ScopeNames.WITH_RUNNER ]).count(query), | ||
254 | RunnerJobModel.scope([ ScopeNames.WITH_RUNNER, ScopeNames.WITH_PARENT ]).findAll<MRunnerJobRunnerParent>(query) | ||
255 | ]).then(([ total, data ]) => ({ total, data })) | ||
256 | } | ||
257 | |||
258 | static updateDependantJobsOf (runnerJob: MRunnerJob) { | ||
259 | const where = { | ||
260 | dependsOnRunnerJobId: runnerJob.id | ||
261 | } | ||
262 | |||
263 | return RunnerJobModel.update({ state: RunnerJobState.PENDING }, { where }) | ||
264 | } | ||
265 | |||
266 | static cancelAllJobs (options: { type: RunnerJobType }) { | ||
267 | const where = { | ||
268 | type: options.type | ||
269 | } | ||
270 | |||
271 | return RunnerJobModel.update({ state: RunnerJobState.CANCELLED }, { where }) | ||
272 | } | ||
273 | |||
274 | // --------------------------------------------------------------------------- | ||
275 | |||
276 | resetToPending () { | ||
277 | this.state = RunnerJobState.PENDING | ||
278 | this.processingJobToken = null | ||
279 | this.progress = null | ||
280 | this.startedAt = null | ||
281 | this.runnerId = null | ||
282 | } | ||
283 | |||
284 | setToErrorOrCancel ( | ||
285 | state: RunnerJobState.PARENT_ERRORED | RunnerJobState.ERRORED | RunnerJobState.CANCELLED | RunnerJobState.PARENT_CANCELLED | ||
286 | ) { | ||
287 | this.state = state | ||
288 | this.processingJobToken = null | ||
289 | this.finishedAt = new Date() | ||
290 | } | ||
291 | |||
292 | toFormattedJSON (this: MRunnerJobRunnerParent): RunnerJob { | ||
293 | const runner = this.Runner | ||
294 | ? { | ||
295 | id: this.Runner.id, | ||
296 | name: this.Runner.name, | ||
297 | description: this.Runner.description | ||
298 | } | ||
299 | : null | ||
300 | |||
301 | const parent = this.DependsOnRunnerJob | ||
302 | ? { | ||
303 | id: this.DependsOnRunnerJob.id, | ||
304 | uuid: this.DependsOnRunnerJob.uuid, | ||
305 | type: this.DependsOnRunnerJob.type, | ||
306 | state: { | ||
307 | id: this.DependsOnRunnerJob.state, | ||
308 | label: RUNNER_JOB_STATES[this.DependsOnRunnerJob.state] | ||
309 | } | ||
310 | } | ||
311 | : undefined | ||
312 | |||
313 | return { | ||
314 | uuid: this.uuid, | ||
315 | type: this.type, | ||
316 | |||
317 | state: { | ||
318 | id: this.state, | ||
319 | label: RUNNER_JOB_STATES[this.state] | ||
320 | }, | ||
321 | |||
322 | progress: this.progress, | ||
323 | priority: this.priority, | ||
324 | failures: this.failures, | ||
325 | error: this.error, | ||
326 | |||
327 | payload: this.payload, | ||
328 | |||
329 | startedAt: this.startedAt?.toISOString(), | ||
330 | finishedAt: this.finishedAt?.toISOString(), | ||
331 | |||
332 | createdAt: this.createdAt.toISOString(), | ||
333 | updatedAt: this.updatedAt.toISOString(), | ||
334 | |||
335 | parent, | ||
336 | runner | ||
337 | } | ||
338 | } | ||
339 | |||
340 | toFormattedAdminJSON (this: MRunnerJobRunnerParent): RunnerJobAdmin { | ||
341 | return { | ||
342 | ...this.toFormattedJSON(), | ||
343 | |||
344 | privatePayload: this.privatePayload | ||
345 | } | ||
346 | } | ||
347 | } | ||
diff --git a/server/models/runner/runner-registration-token.ts b/server/models/runner/runner-registration-token.ts new file mode 100644 index 000000000..b2ae6c9eb --- /dev/null +++ b/server/models/runner/runner-registration-token.ts | |||
@@ -0,0 +1,103 @@ | |||
1 | import { FindOptions, literal } from 'sequelize' | ||
2 | import { AllowNull, Column, CreatedAt, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript' | ||
3 | import { MRunnerRegistrationToken } from '@server/types/models/runners' | ||
4 | import { RunnerRegistrationToken } from '@shared/models' | ||
5 | import { AttributesOnly } from '@shared/typescript-utils' | ||
6 | import { getSort } from '../shared' | ||
7 | import { RunnerModel } from './runner' | ||
8 | |||
9 | /** | ||
10 | * | ||
11 | * Tokens used by PeerTube runners to register themselves to the PeerTube instance | ||
12 | * | ||
13 | */ | ||
14 | |||
15 | @Table({ | ||
16 | tableName: 'runnerRegistrationToken', | ||
17 | indexes: [ | ||
18 | { | ||
19 | fields: [ 'registrationToken' ], | ||
20 | unique: true | ||
21 | } | ||
22 | ] | ||
23 | }) | ||
24 | export class RunnerRegistrationTokenModel extends Model<Partial<AttributesOnly<RunnerRegistrationTokenModel>>> { | ||
25 | |||
26 | @AllowNull(false) | ||
27 | @Column | ||
28 | registrationToken: string | ||
29 | |||
30 | @CreatedAt | ||
31 | createdAt: Date | ||
32 | |||
33 | @UpdatedAt | ||
34 | updatedAt: Date | ||
35 | |||
36 | @HasMany(() => RunnerModel, { | ||
37 | foreignKey: { | ||
38 | allowNull: true | ||
39 | }, | ||
40 | onDelete: 'cascade' | ||
41 | }) | ||
42 | Runners: RunnerModel[] | ||
43 | |||
44 | static load (id: number) { | ||
45 | return RunnerRegistrationTokenModel.findByPk(id) | ||
46 | } | ||
47 | |||
48 | static loadByRegistrationToken (registrationToken: string) { | ||
49 | const query = { | ||
50 | where: { registrationToken } | ||
51 | } | ||
52 | |||
53 | return RunnerRegistrationTokenModel.findOne(query) | ||
54 | } | ||
55 | |||
56 | static countTotal () { | ||
57 | return RunnerRegistrationTokenModel.unscoped().count() | ||
58 | } | ||
59 | |||
60 | static listForApi (options: { | ||
61 | start: number | ||
62 | count: number | ||
63 | sort: string | ||
64 | }) { | ||
65 | const { start, count, sort } = options | ||
66 | |||
67 | const query: FindOptions = { | ||
68 | attributes: { | ||
69 | include: [ | ||
70 | [ | ||
71 | literal('(SELECT COUNT(*) FROM "runner" WHERE "runner"."runnerRegistrationTokenId" = "RunnerRegistrationTokenModel"."id")'), | ||
72 | 'registeredRunnersCount' | ||
73 | ] | ||
74 | ] | ||
75 | }, | ||
76 | offset: start, | ||
77 | limit: count, | ||
78 | order: getSort(sort) | ||
79 | } | ||
80 | |||
81 | return Promise.all([ | ||
82 | RunnerRegistrationTokenModel.count(query), | ||
83 | RunnerRegistrationTokenModel.findAll<MRunnerRegistrationToken>(query) | ||
84 | ]).then(([ total, data ]) => ({ total, data })) | ||
85 | } | ||
86 | |||
87 | // --------------------------------------------------------------------------- | ||
88 | |||
89 | toFormattedJSON (this: MRunnerRegistrationToken): RunnerRegistrationToken { | ||
90 | const registeredRunnersCount = this.get('registeredRunnersCount') as number | ||
91 | |||
92 | return { | ||
93 | id: this.id, | ||
94 | |||
95 | registrationToken: this.registrationToken, | ||
96 | |||
97 | createdAt: this.createdAt, | ||
98 | updatedAt: this.updatedAt, | ||
99 | |||
100 | registeredRunnersCount | ||
101 | } | ||
102 | } | ||
103 | } | ||
diff --git a/server/models/runner/runner.ts b/server/models/runner/runner.ts new file mode 100644 index 000000000..1ef0018b4 --- /dev/null +++ b/server/models/runner/runner.ts | |||
@@ -0,0 +1,112 @@ | |||
1 | import { FindOptions } from 'sequelize' | ||
2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' | ||
3 | import { MRunner } from '@server/types/models/runners' | ||
4 | import { Runner } from '@shared/models' | ||
5 | import { AttributesOnly } from '@shared/typescript-utils' | ||
6 | import { getSort } from '../shared' | ||
7 | import { RunnerRegistrationTokenModel } from './runner-registration-token' | ||
8 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
9 | |||
10 | @Table({ | ||
11 | tableName: 'runner', | ||
12 | indexes: [ | ||
13 | { | ||
14 | fields: [ 'runnerToken' ], | ||
15 | unique: true | ||
16 | }, | ||
17 | { | ||
18 | fields: [ 'runnerRegistrationTokenId' ] | ||
19 | } | ||
20 | ] | ||
21 | }) | ||
22 | export class RunnerModel extends Model<Partial<AttributesOnly<RunnerModel>>> { | ||
23 | |||
24 | // Used to identify the appropriate runner when it uses the runner REST API | ||
25 | @AllowNull(false) | ||
26 | @Column | ||
27 | runnerToken: string | ||
28 | |||
29 | @AllowNull(false) | ||
30 | @Column | ||
31 | name: string | ||
32 | |||
33 | @AllowNull(true) | ||
34 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNERS.DESCRIPTION.max)) | ||
35 | description: string | ||
36 | |||
37 | @AllowNull(false) | ||
38 | @Column | ||
39 | lastContact: Date | ||
40 | |||
41 | @AllowNull(false) | ||
42 | @Column | ||
43 | ip: string | ||
44 | |||
45 | @CreatedAt | ||
46 | createdAt: Date | ||
47 | |||
48 | @UpdatedAt | ||
49 | updatedAt: Date | ||
50 | |||
51 | @ForeignKey(() => RunnerRegistrationTokenModel) | ||
52 | @Column | ||
53 | runnerRegistrationTokenId: number | ||
54 | |||
55 | @BelongsTo(() => RunnerRegistrationTokenModel, { | ||
56 | foreignKey: { | ||
57 | allowNull: false | ||
58 | }, | ||
59 | onDelete: 'cascade' | ||
60 | }) | ||
61 | RunnerRegistrationToken: RunnerRegistrationTokenModel | ||
62 | |||
63 | // --------------------------------------------------------------------------- | ||
64 | |||
65 | static load (id: number) { | ||
66 | return RunnerModel.findByPk(id) | ||
67 | } | ||
68 | |||
69 | static loadByToken (runnerToken: string) { | ||
70 | const query = { | ||
71 | where: { runnerToken } | ||
72 | } | ||
73 | |||
74 | return RunnerModel.findOne(query) | ||
75 | } | ||
76 | |||
77 | static listForApi (options: { | ||
78 | start: number | ||
79 | count: number | ||
80 | sort: string | ||
81 | }) { | ||
82 | const { start, count, sort } = options | ||
83 | |||
84 | const query: FindOptions = { | ||
85 | offset: start, | ||
86 | limit: count, | ||
87 | order: getSort(sort) | ||
88 | } | ||
89 | |||
90 | return Promise.all([ | ||
91 | RunnerModel.count(query), | ||
92 | RunnerModel.findAll<MRunner>(query) | ||
93 | ]).then(([ total, data ]) => ({ total, data })) | ||
94 | } | ||
95 | |||
96 | // --------------------------------------------------------------------------- | ||
97 | |||
98 | toFormattedJSON (this: MRunner): Runner { | ||
99 | return { | ||
100 | id: this.id, | ||
101 | |||
102 | name: this.name, | ||
103 | description: this.description, | ||
104 | |||
105 | ip: this.ip, | ||
106 | lastContact: this.lastContact, | ||
107 | |||
108 | createdAt: this.createdAt, | ||
109 | updatedAt: this.updatedAt | ||
110 | } | ||
111 | } | ||
112 | } | ||
diff --git a/server/models/shared/update.ts b/server/models/shared/update.ts index d02c4535d..96db43730 100644 --- a/server/models/shared/update.ts +++ b/server/models/shared/update.ts | |||
@@ -1,22 +1,32 @@ | |||
1 | import { QueryTypes, Sequelize, Transaction } from 'sequelize' | 1 | import { QueryTypes, Sequelize, Transaction } from 'sequelize' |
2 | 2 | ||
3 | const updating = new Set<string>() | ||
4 | |||
3 | // Sequelize always skip the update if we only update updatedAt field | 5 | // Sequelize always skip the update if we only update updatedAt field |
4 | function setAsUpdated (options: { | 6 | async function setAsUpdated (options: { |
5 | sequelize: Sequelize | 7 | sequelize: Sequelize |
6 | table: string | 8 | table: string |
7 | id: number | 9 | id: number |
8 | transaction?: Transaction | 10 | transaction?: Transaction |
9 | }) { | 11 | }) { |
10 | const { sequelize, table, id, transaction } = options | 12 | const { sequelize, table, id, transaction } = options |
13 | const key = table + '-' + id | ||
14 | |||
15 | if (updating.has(key)) return | ||
16 | updating.add(key) | ||
11 | 17 | ||
12 | return sequelize.query( | 18 | try { |
13 | `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`, | 19 | await sequelize.query( |
14 | { | 20 | `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`, |
15 | replacements: { table, id, updatedAt: new Date() }, | 21 | { |
16 | type: QueryTypes.UPDATE, | 22 | replacements: { table, id, updatedAt: new Date() }, |
17 | transaction | 23 | type: QueryTypes.UPDATE, |
18 | } | 24 | transaction |
19 | ) | 25 | } |
26 | ) | ||
27 | } finally { | ||
28 | updating.delete(key) | ||
29 | } | ||
20 | } | 30 | } |
21 | 31 | ||
22 | export { | 32 | export { |
diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts index 740f6b5c6..5845b8c74 100644 --- a/server/models/video/video-job-info.ts +++ b/server/models/video/video-job-info.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import { Op, QueryTypes, Transaction } from 'sequelize' | 1 | import { Op, QueryTypes, Transaction } from 'sequelize' |
2 | import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' | 2 | import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' |
3 | import { forceNumber } from '@shared/core-utils' | ||
3 | import { AttributesOnly } from '@shared/typescript-utils' | 4 | import { AttributesOnly } from '@shared/typescript-utils' |
4 | import { VideoModel } from './video' | 5 | import { VideoModel } from './video' |
5 | 6 | ||
@@ -59,32 +60,33 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo | |||
59 | return VideoJobInfoModel.findOne({ where, transaction }) | 60 | return VideoJobInfoModel.findOne({ where, transaction }) |
60 | } | 61 | } |
61 | 62 | ||
62 | static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { | 63 | static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType, amountArg = 1): Promise<number> { |
63 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | 64 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } |
65 | const amount = forceNumber(amountArg) | ||
64 | 66 | ||
65 | const [ { pendingMove } ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | 67 | const [ result ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` |
66 | INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") | 68 | INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") |
67 | SELECT | 69 | SELECT |
68 | "video"."id" AS "videoId", 1, NOW(), NOW() | 70 | "video"."id" AS "videoId", ${amount}, NOW(), NOW() |
69 | FROM | 71 | FROM |
70 | "video" | 72 | "video" |
71 | WHERE | 73 | WHERE |
72 | "video"."uuid" = $videoUUID | 74 | "video"."uuid" = $videoUUID |
73 | ON CONFLICT ("videoId") DO UPDATE | 75 | ON CONFLICT ("videoId") DO UPDATE |
74 | SET | 76 | SET |
75 | "${column}" = "videoJobInfo"."${column}" + 1, | 77 | "${column}" = "videoJobInfo"."${column}" + ${amount}, |
76 | "updatedAt" = NOW() | 78 | "updatedAt" = NOW() |
77 | RETURNING | 79 | RETURNING |
78 | "${column}" | 80 | "${column}" |
79 | `, options) | 81 | `, options) |
80 | 82 | ||
81 | return pendingMove | 83 | return result[column] |
82 | } | 84 | } |
83 | 85 | ||
84 | static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { | 86 | static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise<number> { |
85 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } | 87 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } |
86 | 88 | ||
87 | const result = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` | 89 | const result = await VideoJobInfoModel.sequelize.query(` |
88 | UPDATE | 90 | UPDATE |
89 | "videoJobInfo" | 91 | "videoJobInfo" |
90 | SET | 92 | SET |
@@ -99,7 +101,7 @@ export class VideoJobInfoModel extends Model<Partial<AttributesOnly<VideoJobInfo | |||
99 | 101 | ||
100 | if (result.length === 0) return undefined | 102 | if (result.length === 0) return undefined |
101 | 103 | ||
102 | return result[0].pendingMove | 104 | return result[0][column] |
103 | } | 105 | } |
104 | 106 | ||
105 | static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> { | 107 | static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise<void> { |
diff --git a/server/models/video/video-live-session.ts b/server/models/video/video-live-session.ts index dcded7872..9426f5d11 100644 --- a/server/models/video/video-live-session.ts +++ b/server/models/video/video-live-session.ts | |||
@@ -147,12 +147,21 @@ export class VideoLiveSessionModel extends Model<Partial<AttributesOnly<VideoLiv | |||
147 | return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findOne(query) | 147 | return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findOne(query) |
148 | } | 148 | } |
149 | 149 | ||
150 | static findCurrentSessionOf (videoId: number) { | 150 | static findCurrentSessionOf (videoUUID: string) { |
151 | return VideoLiveSessionModel.findOne({ | 151 | return VideoLiveSessionModel.findOne({ |
152 | where: { | 152 | where: { |
153 | liveVideoId: videoId, | ||
154 | endDate: null | 153 | endDate: null |
155 | }, | 154 | }, |
155 | include: [ | ||
156 | { | ||
157 | model: VideoModel.unscoped(), | ||
158 | as: 'LiveVideo', | ||
159 | required: true, | ||
160 | where: { | ||
161 | uuid: videoUUID | ||
162 | } | ||
163 | } | ||
164 | ], | ||
156 | order: [ [ 'startDate', 'DESC' ] ] | 165 | order: [ [ 'startDate', 'DESC' ] ] |
157 | }) | 166 | }) |
158 | } | 167 | } |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index f817c4a33..baa8c120a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -29,12 +29,14 @@ import { LiveManager } from '@server/lib/live/live-manager' | |||
29 | import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' | 29 | import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' |
30 | import { tracer } from '@server/lib/opentelemetry/tracing' | 30 | import { tracer } from '@server/lib/opentelemetry/tracing' |
31 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | 31 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
32 | import { Hooks } from '@server/lib/plugins/hooks' | ||
32 | import { VideoPathManager } from '@server/lib/video-path-manager' | 33 | import { VideoPathManager } from '@server/lib/video-path-manager' |
33 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' | 34 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' |
34 | import { getServerActor } from '@server/models/application/application' | 35 | import { getServerActor } from '@server/models/application/application' |
35 | import { ModelCache } from '@server/models/shared/model-cache' | 36 | import { ModelCache } from '@server/models/shared/model-cache' |
36 | import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' | 37 | import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' |
37 | import { ffprobePromise, getAudioStream, hasAudioStream, uuidToShort } from '@shared/extra-utils' | 38 | import { uuidToShort } from '@shared/extra-utils' |
39 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' | ||
38 | import { | 40 | import { |
39 | ResultList, | 41 | ResultList, |
40 | ThumbnailType, | 42 | ThumbnailType, |
@@ -62,7 +64,6 @@ import { | |||
62 | isVideoStateValid, | 64 | isVideoStateValid, |
63 | isVideoSupportValid | 65 | isVideoSupportValid |
64 | } from '../../helpers/custom-validators/videos' | 66 | } from '../../helpers/custom-validators/videos' |
65 | import { getVideoStreamDimensionsInfo } from '../../helpers/ffmpeg' | ||
66 | import { logger } from '../../helpers/logger' | 67 | import { logger } from '../../helpers/logger' |
67 | import { CONFIG } from '../../initializers/config' | 68 | import { CONFIG } from '../../initializers/config' |
68 | import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants' | 69 | import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants' |
@@ -137,7 +138,6 @@ import { VideoShareModel } from './video-share' | |||
137 | import { VideoSourceModel } from './video-source' | 138 | import { VideoSourceModel } from './video-source' |
138 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | 139 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' |
139 | import { VideoTagModel } from './video-tag' | 140 | import { VideoTagModel } from './video-tag' |
140 | import { Hooks } from '@server/lib/plugins/hooks' | ||
141 | 141 | ||
142 | export enum ScopeNames { | 142 | export enum ScopeNames { |
143 | FOR_API = 'FOR_API', | 143 | FOR_API = 'FOR_API', |
@@ -798,7 +798,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
798 | 798 | ||
799 | logger.info('Stopping live of video %s after video deletion.', instance.uuid) | 799 | logger.info('Stopping live of video %s after video deletion.', instance.uuid) |
800 | 800 | ||
801 | LiveManager.Instance.stopSessionOf(instance.id, null) | 801 | LiveManager.Instance.stopSessionOf(instance.uuid, null) |
802 | } | 802 | } |
803 | 803 | ||
804 | @BeforeDestroy | 804 | @BeforeDestroy |
@@ -1763,10 +1763,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> { | |||
1763 | 1763 | ||
1764 | const { audioStream } = await getAudioStream(originalFilePath, probe) | 1764 | const { audioStream } = await getAudioStream(originalFilePath, probe) |
1765 | const hasAudio = await hasAudioStream(originalFilePath, probe) | 1765 | const hasAudio = await hasAudioStream(originalFilePath, probe) |
1766 | const fps = await getVideoStreamFPS(originalFilePath, probe) | ||
1766 | 1767 | ||
1767 | return { | 1768 | return { |
1768 | audioStream, | 1769 | audioStream, |
1769 | hasAudio, | 1770 | hasAudio, |
1771 | fps, | ||
1770 | 1772 | ||
1771 | ...await getVideoStreamDimensionsInfo(originalFilePath, probe) | 1773 | ...await getVideoStreamDimensionsInfo(originalFilePath, probe) |
1772 | } | 1774 | } |
diff --git a/server/types/express.d.ts b/server/types/express.d.ts index a992a9926..a8aeabb3a 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts | |||
@@ -44,6 +44,7 @@ import { | |||
44 | MVideoShareActor, | 44 | MVideoShareActor, |
45 | MVideoThumbnail | 45 | MVideoThumbnail |
46 | } from './models' | 46 | } from './models' |
47 | import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners' | ||
47 | import { MVideoSource } from './models/video/video-source' | 48 | import { MVideoSource } from './models/video/video-source' |
48 | 49 | ||
49 | declare module 'express' { | 50 | declare module 'express' { |
@@ -102,6 +103,8 @@ declare module 'express' { | |||
102 | instance?: string | 103 | instance?: string |
103 | 104 | ||
104 | data?: PeerTubeProblemDocumentData | 105 | data?: PeerTubeProblemDocumentData |
106 | |||
107 | tags?: string[] | ||
105 | }) => void | 108 | }) => void |
106 | 109 | ||
107 | locals: { | 110 | locals: { |
@@ -203,6 +206,9 @@ declare module 'express' { | |||
203 | 206 | ||
204 | localViewerFull?: MLocalVideoViewerWithWatchSections | 207 | localViewerFull?: MLocalVideoViewerWithWatchSections |
205 | 208 | ||
209 | runner?: MRunner | ||
210 | runnerRegistrationToken?: MRunnerRegistrationToken | ||
211 | runnerJob?: MRunnerJobRunner | ||
206 | } | 212 | } |
207 | } | 213 | } |
208 | } | 214 | } |
diff --git a/server/types/models/runners/index.ts b/server/types/models/runners/index.ts new file mode 100644 index 000000000..e94d4794e --- /dev/null +++ b/server/types/models/runners/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export * from './runner' | ||
2 | export * from './runner-job' | ||
3 | export * from './runner-registration-token' | ||
diff --git a/server/types/models/runners/runner-job.ts b/server/types/models/runners/runner-job.ts new file mode 100644 index 000000000..ec983ba32 --- /dev/null +++ b/server/types/models/runners/runner-job.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import { RunnerJobModel } from '@server/models/runner/runner-job' | ||
2 | import { PickWith } from '@shared/typescript-utils' | ||
3 | import { MRunner } from './runner' | ||
4 | |||
5 | type Use<K extends keyof RunnerJobModel, M> = PickWith<RunnerJobModel, K, M> | ||
6 | |||
7 | // ############################################################################ | ||
8 | |||
9 | export type MRunnerJob = Omit<RunnerJobModel, 'Runner' | 'DependsOnRunnerJob'> | ||
10 | |||
11 | // ############################################################################ | ||
12 | |||
13 | export type MRunnerJobRunner = | ||
14 | MRunnerJob & | ||
15 | Use<'Runner', MRunner> | ||
16 | |||
17 | export type MRunnerJobRunnerParent = | ||
18 | MRunnerJob & | ||
19 | Use<'Runner', MRunner> & | ||
20 | Use<'DependsOnRunnerJob', MRunnerJob> | ||
diff --git a/server/types/models/runners/runner-registration-token.ts b/server/types/models/runners/runner-registration-token.ts new file mode 100644 index 000000000..83b8614ad --- /dev/null +++ b/server/types/models/runners/runner-registration-token.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' | ||
2 | |||
3 | // ############################################################################ | ||
4 | |||
5 | export type MRunnerRegistrationToken = Omit<RunnerRegistrationTokenModel, 'Runners'> | ||
diff --git a/server/types/models/runners/runner.ts b/server/types/models/runners/runner.ts new file mode 100644 index 000000000..d35356378 --- /dev/null +++ b/server/types/models/runners/runner.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | import { RunnerModel } from '@server/models/runner/runner' | ||
2 | |||
3 | // ############################################################################ | ||
4 | |||
5 | export type MRunner = Omit<RunnerModel, 'RunnerRegistrationToken'> | ||
diff --git a/shared/core-utils/common/number.ts b/shared/core-utils/common/number.ts index 9a96dcf5c..ce5a6041a 100644 --- a/shared/core-utils/common/number.ts +++ b/shared/core-utils/common/number.ts | |||
@@ -1,7 +1,13 @@ | |||
1 | function forceNumber (value: any) { | 1 | export function forceNumber (value: any) { |
2 | return parseInt(value + '') | 2 | return parseInt(value + '') |
3 | } | 3 | } |
4 | 4 | ||
5 | export { | 5 | export function isOdd (num: number) { |
6 | forceNumber | 6 | return (num % 2) !== 0 |
7 | } | ||
8 | |||
9 | export function toEven (num: number) { | ||
10 | if (isOdd(num)) return num + 1 | ||
11 | |||
12 | return num | ||
7 | } | 13 | } |
diff --git a/shared/core-utils/common/promises.ts b/shared/core-utils/common/promises.ts index f17221b97..e3792d12e 100644 --- a/shared/core-utils/common/promises.ts +++ b/shared/core-utils/common/promises.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | function isPromise <T = unknown> (value: T | Promise<T>): value is Promise<T> { | 1 | export function isPromise <T = unknown> (value: T | Promise<T>): value is Promise<T> { |
2 | return value && typeof (value as Promise<T>).then === 'function' | 2 | return value && typeof (value as Promise<T>).then === 'function' |
3 | } | 3 | } |
4 | 4 | ||
5 | function isCatchable (value: any) { | 5 | export function isCatchable (value: any) { |
6 | return value && typeof value.catch === 'function' | 6 | return value && typeof value.catch === 'function' |
7 | } | 7 | } |
8 | 8 | ||
9 | function timeoutPromise <T> (promise: Promise<T>, timeoutMs: number) { | 9 | export function timeoutPromise <T> (promise: Promise<T>, timeoutMs: number) { |
10 | let timer: ReturnType<typeof setTimeout> | 10 | let timer: ReturnType<typeof setTimeout> |
11 | 11 | ||
12 | return Promise.race([ | 12 | return Promise.race([ |
@@ -18,8 +18,41 @@ function timeoutPromise <T> (promise: Promise<T>, timeoutMs: number) { | |||
18 | ]).finally(() => clearTimeout(timer)) | 18 | ]).finally(() => clearTimeout(timer)) |
19 | } | 19 | } |
20 | 20 | ||
21 | export { | 21 | export function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { |
22 | isPromise, | 22 | return function promisified (): Promise<A> { |
23 | isCatchable, | 23 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { |
24 | timeoutPromise | 24 | // eslint-disable-next-line no-useless-call |
25 | func.apply(null, [ (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
26 | }) | ||
27 | } | ||
28 | } | ||
29 | |||
30 | // Thanks to https://gist.github.com/kumasento/617daa7e46f13ecdd9b2 | ||
31 | export function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) => void): (arg: T) => Promise<A> { | ||
32 | return function promisified (arg: T): Promise<A> { | ||
33 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
34 | // eslint-disable-next-line no-useless-call | ||
35 | func.apply(null, [ arg, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
36 | }) | ||
37 | } | ||
38 | } | ||
39 | |||
40 | // eslint-disable-next-line max-len | ||
41 | export function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> { | ||
42 | return function promisified (arg1: T, arg2: U): Promise<A> { | ||
43 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
44 | // eslint-disable-next-line no-useless-call | ||
45 | func.apply(null, [ arg1, arg2, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
46 | }) | ||
47 | } | ||
48 | } | ||
49 | |||
50 | // eslint-disable-next-line max-len | ||
51 | export function promisify3<T, U, V, A> (func: (arg1: T, arg2: U, arg3: V, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U, arg3: V) => Promise<A> { | ||
52 | return function promisified (arg1: T, arg2: U, arg3: V): Promise<A> { | ||
53 | return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { | ||
54 | // eslint-disable-next-line no-useless-call | ||
55 | func.apply(null, [ arg1, arg2, arg3, (err: any, res: A) => err ? reject(err) : resolve(res) ]) | ||
56 | }) | ||
57 | } | ||
25 | } | 58 | } |
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts index e2e161a7b..d4cfcbec8 100644 --- a/shared/extra-utils/index.ts +++ b/shared/extra-utils/index.ts | |||
@@ -1,4 +1,3 @@ | |||
1 | export * from './crypto' | 1 | export * from './crypto' |
2 | export * from './ffprobe' | ||
3 | export * from './file' | 2 | export * from './file' |
4 | export * from './uuid' | 3 | export * from './uuid' |
diff --git a/shared/ffmpeg/ffmpeg-command-wrapper.ts b/shared/ffmpeg/ffmpeg-command-wrapper.ts new file mode 100644 index 000000000..7a8c19d4b --- /dev/null +++ b/shared/ffmpeg/ffmpeg-command-wrapper.ts | |||
@@ -0,0 +1,234 @@ | |||
1 | import ffmpeg, { FfmpegCommand, getAvailableEncoders } from 'fluent-ffmpeg' | ||
2 | import { pick, promisify0 } from '@shared/core-utils' | ||
3 | import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models' | ||
4 | |||
5 | type FFmpegLogger = { | ||
6 | info: (msg: string, obj?: any) => void | ||
7 | debug: (msg: string, obj?: any) => void | ||
8 | warn: (msg: string, obj?: any) => void | ||
9 | error: (msg: string, obj?: any) => void | ||
10 | } | ||
11 | |||
12 | export interface FFmpegCommandWrapperOptions { | ||
13 | availableEncoders?: AvailableEncoders | ||
14 | profile?: string | ||
15 | |||
16 | niceness: number | ||
17 | tmpDirectory: string | ||
18 | threads: number | ||
19 | |||
20 | logger: FFmpegLogger | ||
21 | lTags?: { tags: string[] } | ||
22 | |||
23 | updateJobProgress?: (progress?: number) => void | ||
24 | } | ||
25 | |||
26 | export class FFmpegCommandWrapper { | ||
27 | private static supportedEncoders: Map<string, boolean> | ||
28 | |||
29 | private readonly availableEncoders: AvailableEncoders | ||
30 | private readonly profile: string | ||
31 | |||
32 | private readonly niceness: number | ||
33 | private readonly tmpDirectory: string | ||
34 | private readonly threads: number | ||
35 | |||
36 | private readonly logger: FFmpegLogger | ||
37 | private readonly lTags: { tags: string[] } | ||
38 | |||
39 | private readonly updateJobProgress: (progress?: number) => void | ||
40 | |||
41 | private command: FfmpegCommand | ||
42 | |||
43 | constructor (options: FFmpegCommandWrapperOptions) { | ||
44 | this.availableEncoders = options.availableEncoders | ||
45 | this.profile = options.profile | ||
46 | this.niceness = options.niceness | ||
47 | this.tmpDirectory = options.tmpDirectory | ||
48 | this.threads = options.threads | ||
49 | this.logger = options.logger | ||
50 | this.lTags = options.lTags || { tags: [] } | ||
51 | this.updateJobProgress = options.updateJobProgress | ||
52 | } | ||
53 | |||
54 | getAvailableEncoders () { | ||
55 | return this.availableEncoders | ||
56 | } | ||
57 | |||
58 | getProfile () { | ||
59 | return this.profile | ||
60 | } | ||
61 | |||
62 | getCommand () { | ||
63 | return this.command | ||
64 | } | ||
65 | |||
66 | // --------------------------------------------------------------------------- | ||
67 | |||
68 | debugLog (msg: string, meta: any) { | ||
69 | this.logger.debug(msg, { ...meta, ...this.lTags }) | ||
70 | } | ||
71 | |||
72 | // --------------------------------------------------------------------------- | ||
73 | |||
74 | buildCommand (input: string) { | ||
75 | if (this.command) throw new Error('Command is already built') | ||
76 | |||
77 | // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems | ||
78 | this.command = ffmpeg(input, { | ||
79 | niceness: this.niceness, | ||
80 | cwd: this.tmpDirectory | ||
81 | }) | ||
82 | |||
83 | if (this.threads > 0) { | ||
84 | // If we don't set any threads ffmpeg will chose automatically | ||
85 | this.command.outputOption('-threads ' + this.threads) | ||
86 | } | ||
87 | |||
88 | return this.command | ||
89 | } | ||
90 | |||
91 | async runCommand (options: { | ||
92 | silent?: boolean // false by default | ||
93 | } = {}) { | ||
94 | const { silent = false } = options | ||
95 | |||
96 | return new Promise<void>((res, rej) => { | ||
97 | let shellCommand: string | ||
98 | |||
99 | this.command.on('start', cmdline => { shellCommand = cmdline }) | ||
100 | |||
101 | this.command.on('error', (err, stdout, stderr) => { | ||
102 | if (silent !== true) this.logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...this.lTags }) | ||
103 | |||
104 | rej(err) | ||
105 | }) | ||
106 | |||
107 | this.command.on('end', (stdout, stderr) => { | ||
108 | this.logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...this.lTags }) | ||
109 | |||
110 | res() | ||
111 | }) | ||
112 | |||
113 | if (this.updateJobProgress) { | ||
114 | this.command.on('progress', progress => { | ||
115 | if (!progress.percent) return | ||
116 | |||
117 | // Sometimes ffmpeg returns an invalid progress | ||
118 | let percent = Math.round(progress.percent) | ||
119 | if (percent < 0) percent = 0 | ||
120 | if (percent > 100) percent = 100 | ||
121 | |||
122 | this.updateJobProgress(percent) | ||
123 | }) | ||
124 | } | ||
125 | |||
126 | this.command.run() | ||
127 | }) | ||
128 | } | ||
129 | |||
130 | // --------------------------------------------------------------------------- | ||
131 | |||
132 | static resetSupportedEncoders () { | ||
133 | FFmpegCommandWrapper.supportedEncoders = undefined | ||
134 | } | ||
135 | |||
136 | // Run encoder builder depending on available encoders | ||
137 | // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one | ||
138 | // If the default one does not exist, check the next encoder | ||
139 | async getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { | ||
140 | streamType: 'video' | 'audio' | ||
141 | input: string | ||
142 | |||
143 | videoType: 'vod' | 'live' | ||
144 | }) { | ||
145 | if (!this.availableEncoders) { | ||
146 | throw new Error('There is no available encoders') | ||
147 | } | ||
148 | |||
149 | const { streamType, videoType } = options | ||
150 | |||
151 | const encodersToTry = this.availableEncoders.encodersToTry[videoType][streamType] | ||
152 | const encoders = this.availableEncoders.available[videoType] | ||
153 | |||
154 | for (const encoder of encodersToTry) { | ||
155 | if (!(await this.checkFFmpegEncoders(this.availableEncoders)).get(encoder)) { | ||
156 | this.logger.debug(`Encoder ${encoder} not available in ffmpeg, skipping.`, this.lTags) | ||
157 | continue | ||
158 | } | ||
159 | |||
160 | if (!encoders[encoder]) { | ||
161 | this.logger.debug(`Encoder ${encoder} not available in peertube encoders, skipping.`, this.lTags) | ||
162 | continue | ||
163 | } | ||
164 | |||
165 | // An object containing available profiles for this encoder | ||
166 | const builderProfiles: EncoderProfile<EncoderOptionsBuilder> = encoders[encoder] | ||
167 | let builder = builderProfiles[this.profile] | ||
168 | |||
169 | if (!builder) { | ||
170 | this.logger.debug(`Profile ${this.profile} for encoder ${encoder} not available. Fallback to default.`, this.lTags) | ||
171 | builder = builderProfiles.default | ||
172 | |||
173 | if (!builder) { | ||
174 | this.logger.debug(`Default profile for encoder ${encoder} not available. Try next available encoder.`, this.lTags) | ||
175 | continue | ||
176 | } | ||
177 | } | ||
178 | |||
179 | const result = await builder( | ||
180 | pick(options, [ | ||
181 | 'input', | ||
182 | 'canCopyAudio', | ||
183 | 'canCopyVideo', | ||
184 | 'resolution', | ||
185 | 'inputBitrate', | ||
186 | 'fps', | ||
187 | 'inputRatio', | ||
188 | 'streamNum' | ||
189 | ]) | ||
190 | ) | ||
191 | |||
192 | return { | ||
193 | result, | ||
194 | |||
195 | // If we don't have output options, then copy the input stream | ||
196 | encoder: result.copy === true | ||
197 | ? 'copy' | ||
198 | : encoder | ||
199 | } | ||
200 | } | ||
201 | |||
202 | return null | ||
203 | } | ||
204 | |||
205 | // Detect supported encoders by ffmpeg | ||
206 | private async checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise<Map<string, boolean>> { | ||
207 | if (FFmpegCommandWrapper.supportedEncoders !== undefined) { | ||
208 | return FFmpegCommandWrapper.supportedEncoders | ||
209 | } | ||
210 | |||
211 | const getAvailableEncodersPromise = promisify0(getAvailableEncoders) | ||
212 | const availableFFmpegEncoders = await getAvailableEncodersPromise() | ||
213 | |||
214 | const searchEncoders = new Set<string>() | ||
215 | for (const type of [ 'live', 'vod' ]) { | ||
216 | for (const streamType of [ 'audio', 'video' ]) { | ||
217 | for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { | ||
218 | searchEncoders.add(encoder) | ||
219 | } | ||
220 | } | ||
221 | } | ||
222 | |||
223 | const supportedEncoders = new Map<string, boolean>() | ||
224 | |||
225 | for (const searchEncoder of searchEncoders) { | ||
226 | supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) | ||
227 | } | ||
228 | |||
229 | this.logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...this.lTags }) | ||
230 | |||
231 | FFmpegCommandWrapper.supportedEncoders = supportedEncoders | ||
232 | return supportedEncoders | ||
233 | } | ||
234 | } | ||
diff --git a/shared/ffmpeg/ffmpeg-edition.ts b/shared/ffmpeg/ffmpeg-edition.ts new file mode 100644 index 000000000..724ca1ea9 --- /dev/null +++ b/shared/ffmpeg/ffmpeg-edition.ts | |||
@@ -0,0 +1,239 @@ | |||
1 | import { FilterSpecification } from 'fluent-ffmpeg' | ||
2 | import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' | ||
3 | import { presetVOD } from './shared/presets' | ||
4 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe' | ||
5 | |||
6 | export class FFmpegEdition { | ||
7 | private readonly commandWrapper: FFmpegCommandWrapper | ||
8 | |||
9 | constructor (options: FFmpegCommandWrapperOptions) { | ||
10 | this.commandWrapper = new FFmpegCommandWrapper(options) | ||
11 | } | ||
12 | |||
13 | async cutVideo (options: { | ||
14 | inputPath: string | ||
15 | outputPath: string | ||
16 | start?: number | ||
17 | end?: number | ||
18 | }) { | ||
19 | const { inputPath, outputPath } = options | ||
20 | |||
21 | const mainProbe = await ffprobePromise(inputPath) | ||
22 | const fps = await getVideoStreamFPS(inputPath, mainProbe) | ||
23 | const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) | ||
24 | |||
25 | const command = this.commandWrapper.buildCommand(inputPath) | ||
26 | .output(outputPath) | ||
27 | |||
28 | await presetVOD({ | ||
29 | commandWrapper: this.commandWrapper, | ||
30 | input: inputPath, | ||
31 | resolution, | ||
32 | fps, | ||
33 | canCopyAudio: false, | ||
34 | canCopyVideo: false | ||
35 | }) | ||
36 | |||
37 | if (options.start) { | ||
38 | command.outputOption('-ss ' + options.start) | ||
39 | } | ||
40 | |||
41 | if (options.end) { | ||
42 | command.outputOption('-to ' + options.end) | ||
43 | } | ||
44 | |||
45 | await this.commandWrapper.runCommand() | ||
46 | } | ||
47 | |||
48 | async addWatermark (options: { | ||
49 | inputPath: string | ||
50 | watermarkPath: string | ||
51 | outputPath: string | ||
52 | |||
53 | videoFilters: { | ||
54 | watermarkSizeRatio: number | ||
55 | horitonzalMarginRatio: number | ||
56 | verticalMarginRatio: number | ||
57 | } | ||
58 | }) { | ||
59 | const { watermarkPath, inputPath, outputPath, videoFilters } = options | ||
60 | |||
61 | const videoProbe = await ffprobePromise(inputPath) | ||
62 | const fps = await getVideoStreamFPS(inputPath, videoProbe) | ||
63 | const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) | ||
64 | |||
65 | const command = this.commandWrapper.buildCommand(inputPath) | ||
66 | .output(outputPath) | ||
67 | |||
68 | command.input(watermarkPath) | ||
69 | |||
70 | await presetVOD({ | ||
71 | commandWrapper: this.commandWrapper, | ||
72 | input: inputPath, | ||
73 | resolution, | ||
74 | fps, | ||
75 | canCopyAudio: true, | ||
76 | canCopyVideo: false | ||
77 | }) | ||
78 | |||
79 | const complexFilter: FilterSpecification[] = [ | ||
80 | // Scale watermark | ||
81 | { | ||
82 | inputs: [ '[1]', '[0]' ], | ||
83 | filter: 'scale2ref', | ||
84 | options: { | ||
85 | w: 'oh*mdar', | ||
86 | h: `ih*${videoFilters.watermarkSizeRatio}` | ||
87 | }, | ||
88 | outputs: [ '[watermark]', '[video]' ] | ||
89 | }, | ||
90 | |||
91 | { | ||
92 | inputs: [ '[video]', '[watermark]' ], | ||
93 | filter: 'overlay', | ||
94 | options: { | ||
95 | x: `main_w - overlay_w - (main_h * ${videoFilters.horitonzalMarginRatio})`, | ||
96 | y: `main_h * ${videoFilters.verticalMarginRatio}` | ||
97 | } | ||
98 | } | ||
99 | ] | ||
100 | |||
101 | command.complexFilter(complexFilter) | ||
102 | |||
103 | await this.commandWrapper.runCommand() | ||
104 | } | ||
105 | |||
106 | async addIntroOutro (options: { | ||
107 | inputPath: string | ||
108 | introOutroPath: string | ||
109 | outputPath: string | ||
110 | type: 'intro' | 'outro' | ||
111 | }) { | ||
112 | const { introOutroPath, inputPath, outputPath, type } = options | ||
113 | |||
114 | const mainProbe = await ffprobePromise(inputPath) | ||
115 | const fps = await getVideoStreamFPS(inputPath, mainProbe) | ||
116 | const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) | ||
117 | const mainHasAudio = await hasAudioStream(inputPath, mainProbe) | ||
118 | |||
119 | const introOutroProbe = await ffprobePromise(introOutroPath) | ||
120 | const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) | ||
121 | |||
122 | const command = this.commandWrapper.buildCommand(inputPath) | ||
123 | .output(outputPath) | ||
124 | |||
125 | command.input(introOutroPath) | ||
126 | |||
127 | if (!introOutroHasAudio && mainHasAudio) { | ||
128 | const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) | ||
129 | |||
130 | command.input('anullsrc') | ||
131 | command.withInputFormat('lavfi') | ||
132 | command.withInputOption('-t ' + duration) | ||
133 | } | ||
134 | |||
135 | await presetVOD({ | ||
136 | commandWrapper: this.commandWrapper, | ||
137 | input: inputPath, | ||
138 | resolution, | ||
139 | fps, | ||
140 | canCopyAudio: false, | ||
141 | canCopyVideo: false | ||
142 | }) | ||
143 | |||
144 | // Add black background to correctly scale intro/outro with padding | ||
145 | const complexFilter: FilterSpecification[] = [ | ||
146 | { | ||
147 | inputs: [ '1', '0' ], | ||
148 | filter: 'scale2ref', | ||
149 | options: { | ||
150 | w: 'iw', | ||
151 | h: `ih` | ||
152 | }, | ||
153 | outputs: [ 'intro-outro', 'main' ] | ||
154 | }, | ||
155 | { | ||
156 | inputs: [ 'intro-outro', 'main' ], | ||
157 | filter: 'scale2ref', | ||
158 | options: { | ||
159 | w: 'iw', | ||
160 | h: `ih` | ||
161 | }, | ||
162 | outputs: [ 'to-scale', 'main' ] | ||
163 | }, | ||
164 | { | ||
165 | inputs: 'to-scale', | ||
166 | filter: 'drawbox', | ||
167 | options: { | ||
168 | t: 'fill' | ||
169 | }, | ||
170 | outputs: [ 'to-scale-bg' ] | ||
171 | }, | ||
172 | { | ||
173 | inputs: [ '1', 'to-scale-bg' ], | ||
174 | filter: 'scale2ref', | ||
175 | options: { | ||
176 | w: 'iw', | ||
177 | h: 'ih', | ||
178 | force_original_aspect_ratio: 'decrease', | ||
179 | flags: 'spline' | ||
180 | }, | ||
181 | outputs: [ 'to-scale', 'to-scale-bg' ] | ||
182 | }, | ||
183 | { | ||
184 | inputs: [ 'to-scale-bg', 'to-scale' ], | ||
185 | filter: 'overlay', | ||
186 | options: { | ||
187 | x: '(main_w - overlay_w)/2', | ||
188 | y: '(main_h - overlay_h)/2' | ||
189 | }, | ||
190 | outputs: 'intro-outro-resized' | ||
191 | } | ||
192 | ] | ||
193 | |||
194 | const concatFilter = { | ||
195 | inputs: [], | ||
196 | filter: 'concat', | ||
197 | options: { | ||
198 | n: 2, | ||
199 | v: 1, | ||
200 | unsafe: 1 | ||
201 | }, | ||
202 | outputs: [ 'v' ] | ||
203 | } | ||
204 | |||
205 | const introOutroFilterInputs = [ 'intro-outro-resized' ] | ||
206 | const mainFilterInputs = [ 'main' ] | ||
207 | |||
208 | if (mainHasAudio) { | ||
209 | mainFilterInputs.push('0:a') | ||
210 | |||
211 | if (introOutroHasAudio) { | ||
212 | introOutroFilterInputs.push('1:a') | ||
213 | } else { | ||
214 | // Silent input | ||
215 | introOutroFilterInputs.push('2:a') | ||
216 | } | ||
217 | } | ||
218 | |||
219 | if (type === 'intro') { | ||
220 | concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] | ||
221 | } else { | ||
222 | concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] | ||
223 | } | ||
224 | |||
225 | if (mainHasAudio) { | ||
226 | concatFilter.options['a'] = 1 | ||
227 | concatFilter.outputs.push('a') | ||
228 | |||
229 | command.outputOption('-map [a]') | ||
230 | } | ||
231 | |||
232 | command.outputOption('-map [v]') | ||
233 | |||
234 | complexFilter.push(concatFilter) | ||
235 | command.complexFilter(complexFilter) | ||
236 | |||
237 | await this.commandWrapper.runCommand() | ||
238 | } | ||
239 | } | ||
diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts new file mode 100644 index 000000000..2db63bd8b --- /dev/null +++ b/shared/ffmpeg/ffmpeg-images.ts | |||
@@ -0,0 +1,59 @@ | |||
1 | import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' | ||
2 | |||
3 | export class FFmpegImage { | ||
4 | private readonly commandWrapper: FFmpegCommandWrapper | ||
5 | |||
6 | constructor (options: FFmpegCommandWrapperOptions) { | ||
7 | this.commandWrapper = new FFmpegCommandWrapper(options) | ||
8 | } | ||
9 | |||
10 | convertWebPToJPG (options: { | ||
11 | path: string | ||
12 | destination: string | ||
13 | }): Promise<void> { | ||
14 | const { path, destination } = options | ||
15 | |||
16 | this.commandWrapper.buildCommand(path) | ||
17 | .output(destination) | ||
18 | |||
19 | return this.commandWrapper.runCommand({ silent: true }) | ||
20 | } | ||
21 | |||
22 | processGIF (options: { | ||
23 | path: string | ||
24 | destination: string | ||
25 | newSize: { width: number, height: number } | ||
26 | }): Promise<void> { | ||
27 | const { path, destination, newSize } = options | ||
28 | |||
29 | this.commandWrapper.buildCommand(path) | ||
30 | .fps(20) | ||
31 | .size(`${newSize.width}x${newSize.height}`) | ||
32 | .output(destination) | ||
33 | |||
34 | return this.commandWrapper.runCommand() | ||
35 | } | ||
36 | |||
37 | async generateThumbnailFromVideo (options: { | ||
38 | fromPath: string | ||
39 | folder: string | ||
40 | imageName: string | ||
41 | }) { | ||
42 | const { fromPath, folder, imageName } = options | ||
43 | |||
44 | const pendingImageName = 'pending-' + imageName | ||
45 | |||
46 | const thumbnailOptions = { | ||
47 | filename: pendingImageName, | ||
48 | count: 1, | ||
49 | folder | ||
50 | } | ||
51 | |||
52 | return new Promise<string>((res, rej) => { | ||
53 | this.commandWrapper.buildCommand(fromPath) | ||
54 | .on('error', rej) | ||
55 | .on('end', () => res(imageName)) | ||
56 | .thumbnail(thumbnailOptions) | ||
57 | }) | ||
58 | } | ||
59 | } | ||
diff --git a/shared/ffmpeg/ffmpeg-live.ts b/shared/ffmpeg/ffmpeg-live.ts new file mode 100644 index 000000000..cca4c6474 --- /dev/null +++ b/shared/ffmpeg/ffmpeg-live.ts | |||
@@ -0,0 +1,184 @@ | |||
1 | import { FilterSpecification } from 'fluent-ffmpeg' | ||
2 | import { join } from 'path' | ||
3 | import { pick } from '@shared/core-utils' | ||
4 | import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' | ||
5 | import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-utils' | ||
6 | import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './shared' | ||
7 | |||
8 | export class FFmpegLive { | ||
9 | private readonly commandWrapper: FFmpegCommandWrapper | ||
10 | |||
11 | constructor (options: FFmpegCommandWrapperOptions) { | ||
12 | this.commandWrapper = new FFmpegCommandWrapper(options) | ||
13 | } | ||
14 | |||
15 | async getLiveTranscodingCommand (options: { | ||
16 | inputUrl: string | ||
17 | |||
18 | outPath: string | ||
19 | masterPlaylistName: string | ||
20 | |||
21 | toTranscode: { | ||
22 | resolution: number | ||
23 | fps: number | ||
24 | }[] | ||
25 | |||
26 | // Input information | ||
27 | bitrate: number | ||
28 | ratio: number | ||
29 | hasAudio: boolean | ||
30 | |||
31 | segmentListSize: number | ||
32 | segmentDuration: number | ||
33 | }) { | ||
34 | const { | ||
35 | inputUrl, | ||
36 | outPath, | ||
37 | toTranscode, | ||
38 | bitrate, | ||
39 | masterPlaylistName, | ||
40 | ratio, | ||
41 | hasAudio | ||
42 | } = options | ||
43 | const command = this.commandWrapper.buildCommand(inputUrl) | ||
44 | |||
45 | const varStreamMap: string[] = [] | ||
46 | |||
47 | const complexFilter: FilterSpecification[] = [ | ||
48 | { | ||
49 | inputs: '[v:0]', | ||
50 | filter: 'split', | ||
51 | options: toTranscode.length, | ||
52 | outputs: toTranscode.map(t => `vtemp${t.resolution}`) | ||
53 | } | ||
54 | ] | ||
55 | |||
56 | command.outputOption('-sc_threshold 0') | ||
57 | |||
58 | addDefaultEncoderGlobalParams(command) | ||
59 | |||
60 | for (let i = 0; i < toTranscode.length; i++) { | ||
61 | const streamMap: string[] = [] | ||
62 | const { resolution, fps } = toTranscode[i] | ||
63 | |||
64 | const baseEncoderBuilderParams = { | ||
65 | input: inputUrl, | ||
66 | |||
67 | canCopyAudio: true, | ||
68 | canCopyVideo: true, | ||
69 | |||
70 | inputBitrate: bitrate, | ||
71 | inputRatio: ratio, | ||
72 | |||
73 | resolution, | ||
74 | fps, | ||
75 | |||
76 | streamNum: i, | ||
77 | videoType: 'live' as 'live' | ||
78 | } | ||
79 | |||
80 | { | ||
81 | const streamType: StreamType = 'video' | ||
82 | const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) | ||
83 | if (!builderResult) { | ||
84 | throw new Error('No available live video encoder found') | ||
85 | } | ||
86 | |||
87 | command.outputOption(`-map [vout${resolution}]`) | ||
88 | |||
89 | addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) | ||
90 | |||
91 | this.commandWrapper.debugLog( | ||
92 | `Apply ffmpeg live video params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, | ||
93 | { builderResult, fps, toTranscode } | ||
94 | ) | ||
95 | |||
96 | command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) | ||
97 | applyEncoderOptions(command, builderResult.result) | ||
98 | |||
99 | complexFilter.push({ | ||
100 | inputs: `vtemp${resolution}`, | ||
101 | filter: getScaleFilter(builderResult.result), | ||
102 | options: `w=-2:h=${resolution}`, | ||
103 | outputs: `vout${resolution}` | ||
104 | }) | ||
105 | |||
106 | streamMap.push(`v:${i}`) | ||
107 | } | ||
108 | |||
109 | if (hasAudio) { | ||
110 | const streamType: StreamType = 'audio' | ||
111 | const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) | ||
112 | if (!builderResult) { | ||
113 | throw new Error('No available live audio encoder found') | ||
114 | } | ||
115 | |||
116 | command.outputOption('-map a:0') | ||
117 | |||
118 | addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) | ||
119 | |||
120 | this.commandWrapper.debugLog( | ||
121 | `Apply ffmpeg live audio params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, | ||
122 | { builderResult, fps, resolution } | ||
123 | ) | ||
124 | |||
125 | command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) | ||
126 | applyEncoderOptions(command, builderResult.result) | ||
127 | |||
128 | streamMap.push(`a:${i}`) | ||
129 | } | ||
130 | |||
131 | varStreamMap.push(streamMap.join(',')) | ||
132 | } | ||
133 | |||
134 | command.complexFilter(complexFilter) | ||
135 | |||
136 | this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) | ||
137 | |||
138 | command.outputOption('-var_stream_map', varStreamMap.join(' ')) | ||
139 | |||
140 | return command | ||
141 | } | ||
142 | |||
143 | getLiveMuxingCommand (options: { | ||
144 | inputUrl: string | ||
145 | outPath: string | ||
146 | masterPlaylistName: string | ||
147 | |||
148 | segmentListSize: number | ||
149 | segmentDuration: number | ||
150 | }) { | ||
151 | const { inputUrl, outPath, masterPlaylistName } = options | ||
152 | |||
153 | const command = this.commandWrapper.buildCommand(inputUrl) | ||
154 | |||
155 | command.outputOption('-c:v copy') | ||
156 | command.outputOption('-c:a copy') | ||
157 | command.outputOption('-map 0:a?') | ||
158 | command.outputOption('-map 0:v?') | ||
159 | |||
160 | this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) | ||
161 | |||
162 | return command | ||
163 | } | ||
164 | |||
165 | private addDefaultLiveHLSParams (options: { | ||
166 | outPath: string | ||
167 | masterPlaylistName: string | ||
168 | segmentListSize: number | ||
169 | segmentDuration: number | ||
170 | }) { | ||
171 | const { outPath, masterPlaylistName, segmentListSize, segmentDuration } = options | ||
172 | |||
173 | const command = this.commandWrapper.getCommand() | ||
174 | |||
175 | command.outputOption('-hls_time ' + segmentDuration) | ||
176 | command.outputOption('-hls_list_size ' + segmentListSize) | ||
177 | command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time') | ||
178 | command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) | ||
179 | command.outputOption('-master_pl_name ' + masterPlaylistName) | ||
180 | command.outputOption(`-f hls`) | ||
181 | |||
182 | command.output(join(outPath, '%v.m3u8')) | ||
183 | } | ||
184 | } | ||
diff --git a/shared/ffmpeg/ffmpeg-utils.ts b/shared/ffmpeg/ffmpeg-utils.ts new file mode 100644 index 000000000..7d09c32ca --- /dev/null +++ b/shared/ffmpeg/ffmpeg-utils.ts | |||
@@ -0,0 +1,17 @@ | |||
1 | import { EncoderOptions } from '@shared/models' | ||
2 | |||
3 | export type StreamType = 'audio' | 'video' | ||
4 | |||
5 | export function buildStreamSuffix (base: string, streamNum?: number) { | ||
6 | if (streamNum !== undefined) { | ||
7 | return `${base}:${streamNum}` | ||
8 | } | ||
9 | |||
10 | return base | ||
11 | } | ||
12 | |||
13 | export function getScaleFilter (options: EncoderOptions): string { | ||
14 | if (options.scaleFilter) return options.scaleFilter.name | ||
15 | |||
16 | return 'scale' | ||
17 | } | ||
diff --git a/shared/ffmpeg/ffmpeg-version.ts b/shared/ffmpeg/ffmpeg-version.ts new file mode 100644 index 000000000..41d9b2d89 --- /dev/null +++ b/shared/ffmpeg/ffmpeg-version.ts | |||
@@ -0,0 +1,24 @@ | |||
1 | import { exec } from 'child_process' | ||
2 | import ffmpeg from 'fluent-ffmpeg' | ||
3 | |||
4 | export function getFFmpegVersion () { | ||
5 | return new Promise<string>((res, rej) => { | ||
6 | (ffmpeg() as any)._getFfmpegPath((err, ffmpegPath) => { | ||
7 | if (err) return rej(err) | ||
8 | if (!ffmpegPath) return rej(new Error('Could not find ffmpeg path')) | ||
9 | |||
10 | return exec(`${ffmpegPath} -version`, (err, stdout) => { | ||
11 | if (err) return rej(err) | ||
12 | |||
13 | const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) | ||
14 | if (!parsed?.[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) | ||
15 | |||
16 | // Fix ffmpeg version that does not include patch version (4.4 for example) | ||
17 | let version = parsed[1] | ||
18 | if (version.match(/^\d+\.\d+$/)) { | ||
19 | version += '.0' | ||
20 | } | ||
21 | }) | ||
22 | }) | ||
23 | }) | ||
24 | } | ||
diff --git a/shared/ffmpeg/ffmpeg-vod.ts b/shared/ffmpeg/ffmpeg-vod.ts new file mode 100644 index 000000000..e40ca0a1e --- /dev/null +++ b/shared/ffmpeg/ffmpeg-vod.ts | |||
@@ -0,0 +1,256 @@ | |||
1 | import { MutexInterface } from 'async-mutex' | ||
2 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
3 | import { readFile, writeFile } from 'fs-extra' | ||
4 | import { dirname } from 'path' | ||
5 | import { pick } from '@shared/core-utils' | ||
6 | import { VideoResolution } from '@shared/models' | ||
7 | import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' | ||
8 | import { ffprobePromise, getVideoStreamDimensionsInfo } from './ffprobe' | ||
9 | import { presetCopy, presetOnlyAudio, presetVOD } from './shared/presets' | ||
10 | |||
11 | export type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' | ||
12 | |||
13 | export interface BaseTranscodeVODOptions { | ||
14 | type: TranscodeVODOptionsType | ||
15 | |||
16 | inputPath: string | ||
17 | outputPath: string | ||
18 | |||
19 | // Will be released after the ffmpeg started | ||
20 | // To prevent a bug where the input file does not exist anymore when running ffmpeg | ||
21 | inputFileMutexReleaser: MutexInterface.Releaser | ||
22 | |||
23 | resolution: number | ||
24 | fps: number | ||
25 | } | ||
26 | |||
27 | export interface HLSTranscodeOptions extends BaseTranscodeVODOptions { | ||
28 | type: 'hls' | ||
29 | |||
30 | copyCodecs: boolean | ||
31 | |||
32 | hlsPlaylist: { | ||
33 | videoFilename: string | ||
34 | } | ||
35 | } | ||
36 | |||
37 | export interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { | ||
38 | type: 'hls-from-ts' | ||
39 | |||
40 | isAAC: boolean | ||
41 | |||
42 | hlsPlaylist: { | ||
43 | videoFilename: string | ||
44 | } | ||
45 | } | ||
46 | |||
47 | export interface QuickTranscodeOptions extends BaseTranscodeVODOptions { | ||
48 | type: 'quick-transcode' | ||
49 | } | ||
50 | |||
51 | export interface VideoTranscodeOptions extends BaseTranscodeVODOptions { | ||
52 | type: 'video' | ||
53 | } | ||
54 | |||
55 | export interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { | ||
56 | type: 'merge-audio' | ||
57 | audioPath: string | ||
58 | } | ||
59 | |||
60 | export interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { | ||
61 | type: 'only-audio' | ||
62 | } | ||
63 | |||
64 | export type TranscodeVODOptions = | ||
65 | HLSTranscodeOptions | ||
66 | | HLSFromTSTranscodeOptions | ||
67 | | VideoTranscodeOptions | ||
68 | | MergeAudioTranscodeOptions | ||
69 | | OnlyAudioTranscodeOptions | ||
70 | | QuickTranscodeOptions | ||
71 | |||
72 | // --------------------------------------------------------------------------- | ||
73 | |||
74 | export class FFmpegVOD { | ||
75 | private readonly commandWrapper: FFmpegCommandWrapper | ||
76 | |||
77 | private ended = false | ||
78 | |||
79 | constructor (options: FFmpegCommandWrapperOptions) { | ||
80 | this.commandWrapper = new FFmpegCommandWrapper(options) | ||
81 | } | ||
82 | |||
83 | async transcode (options: TranscodeVODOptions) { | ||
84 | const builders: { | ||
85 | [ type in TranscodeVODOptionsType ]: (options: TranscodeVODOptions) => Promise<void> | void | ||
86 | } = { | ||
87 | 'quick-transcode': this.buildQuickTranscodeCommand.bind(this), | ||
88 | 'hls': this.buildHLSVODCommand.bind(this), | ||
89 | 'hls-from-ts': this.buildHLSVODFromTSCommand.bind(this), | ||
90 | 'merge-audio': this.buildAudioMergeCommand.bind(this), | ||
91 | // TODO: remove, we merge this in buildWebVideoCommand | ||
92 | 'only-audio': this.buildOnlyAudioCommand.bind(this), | ||
93 | 'video': this.buildWebVideoCommand.bind(this) | ||
94 | } | ||
95 | |||
96 | this.commandWrapper.debugLog('Will run transcode.', { options }) | ||
97 | |||
98 | const command = this.commandWrapper.buildCommand(options.inputPath) | ||
99 | .output(options.outputPath) | ||
100 | |||
101 | await builders[options.type](options) | ||
102 | |||
103 | command.on('start', () => { | ||
104 | setTimeout(() => { | ||
105 | options.inputFileMutexReleaser() | ||
106 | }, 1000) | ||
107 | }) | ||
108 | |||
109 | await this.commandWrapper.runCommand() | ||
110 | |||
111 | await this.fixHLSPlaylistIfNeeded(options) | ||
112 | |||
113 | this.ended = true | ||
114 | } | ||
115 | |||
116 | isEnded () { | ||
117 | return this.ended | ||
118 | } | ||
119 | |||
120 | private async buildWebVideoCommand (options: TranscodeVODOptions) { | ||
121 | const { resolution, fps, inputPath } = options | ||
122 | |||
123 | if (resolution === VideoResolution.H_NOVIDEO) { | ||
124 | presetOnlyAudio(this.commandWrapper) | ||
125 | return | ||
126 | } | ||
127 | |||
128 | let scaleFilterValue: string | ||
129 | |||
130 | if (resolution !== undefined) { | ||
131 | const probe = await ffprobePromise(inputPath) | ||
132 | const videoStreamInfo = await getVideoStreamDimensionsInfo(inputPath, probe) | ||
133 | |||
134 | scaleFilterValue = videoStreamInfo?.isPortraitMode === true | ||
135 | ? `w=${resolution}:h=-2` | ||
136 | : `w=-2:h=${resolution}` | ||
137 | } | ||
138 | |||
139 | await presetVOD({ | ||
140 | commandWrapper: this.commandWrapper, | ||
141 | |||
142 | resolution, | ||
143 | input: inputPath, | ||
144 | canCopyAudio: true, | ||
145 | canCopyVideo: true, | ||
146 | fps, | ||
147 | scaleFilterValue | ||
148 | }) | ||
149 | } | ||
150 | |||
151 | private buildQuickTranscodeCommand (_options: TranscodeVODOptions) { | ||
152 | const command = this.commandWrapper.getCommand() | ||
153 | |||
154 | presetCopy(this.commandWrapper) | ||
155 | |||
156 | command.outputOption('-map_metadata -1') // strip all metadata | ||
157 | .outputOption('-movflags faststart') | ||
158 | } | ||
159 | |||
160 | // --------------------------------------------------------------------------- | ||
161 | // Audio transcoding | ||
162 | // --------------------------------------------------------------------------- | ||
163 | |||
164 | private async buildAudioMergeCommand (options: MergeAudioTranscodeOptions) { | ||
165 | const command = this.commandWrapper.getCommand() | ||
166 | |||
167 | command.loop(undefined) | ||
168 | |||
169 | await presetVOD({ | ||
170 | ...pick(options, [ 'resolution' ]), | ||
171 | |||
172 | commandWrapper: this.commandWrapper, | ||
173 | input: options.audioPath, | ||
174 | canCopyAudio: true, | ||
175 | canCopyVideo: true, | ||
176 | fps: options.fps, | ||
177 | scaleFilterValue: this.getMergeAudioScaleFilterValue() | ||
178 | }) | ||
179 | |||
180 | command.outputOption('-preset:v veryfast') | ||
181 | |||
182 | command.input(options.audioPath) | ||
183 | .outputOption('-tune stillimage') | ||
184 | .outputOption('-shortest') | ||
185 | } | ||
186 | |||
187 | private buildOnlyAudioCommand (_options: OnlyAudioTranscodeOptions) { | ||
188 | presetOnlyAudio(this.commandWrapper) | ||
189 | } | ||
190 | |||
191 | // Avoid "height not divisible by 2" error | ||
192 | private getMergeAudioScaleFilterValue () { | ||
193 | return 'trunc(iw/2)*2:trunc(ih/2)*2' | ||
194 | } | ||
195 | |||
196 | // --------------------------------------------------------------------------- | ||
197 | // HLS transcoding | ||
198 | // --------------------------------------------------------------------------- | ||
199 | |||
200 | private async buildHLSVODCommand (options: HLSTranscodeOptions) { | ||
201 | const command = this.commandWrapper.getCommand() | ||
202 | |||
203 | const videoPath = this.getHLSVideoPath(options) | ||
204 | |||
205 | if (options.copyCodecs) presetCopy(this.commandWrapper) | ||
206 | else if (options.resolution === VideoResolution.H_NOVIDEO) presetOnlyAudio(this.commandWrapper) | ||
207 | else await this.buildWebVideoCommand(options) | ||
208 | |||
209 | this.addCommonHLSVODCommandOptions(command, videoPath) | ||
210 | } | ||
211 | |||
212 | private buildHLSVODFromTSCommand (options: HLSFromTSTranscodeOptions) { | ||
213 | const command = this.commandWrapper.getCommand() | ||
214 | |||
215 | const videoPath = this.getHLSVideoPath(options) | ||
216 | |||
217 | command.outputOption('-c copy') | ||
218 | |||
219 | if (options.isAAC) { | ||
220 | // Required for example when copying an AAC stream from an MPEG-TS | ||
221 | // Since it's a bitstream filter, we don't need to reencode the audio | ||
222 | command.outputOption('-bsf:a aac_adtstoasc') | ||
223 | } | ||
224 | |||
225 | this.addCommonHLSVODCommandOptions(command, videoPath) | ||
226 | } | ||
227 | |||
228 | private addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { | ||
229 | return command.outputOption('-hls_time 4') | ||
230 | .outputOption('-hls_list_size 0') | ||
231 | .outputOption('-hls_playlist_type vod') | ||
232 | .outputOption('-hls_segment_filename ' + outputPath) | ||
233 | .outputOption('-hls_segment_type fmp4') | ||
234 | .outputOption('-f hls') | ||
235 | .outputOption('-hls_flags single_file') | ||
236 | } | ||
237 | |||
238 | private async fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { | ||
239 | if (options.type !== 'hls' && options.type !== 'hls-from-ts') return | ||
240 | |||
241 | const fileContent = await readFile(options.outputPath) | ||
242 | |||
243 | const videoFileName = options.hlsPlaylist.videoFilename | ||
244 | const videoFilePath = this.getHLSVideoPath(options) | ||
245 | |||
246 | // Fix wrong mapping with some ffmpeg versions | ||
247 | const newContent = fileContent.toString() | ||
248 | .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) | ||
249 | |||
250 | await writeFile(options.outputPath, newContent) | ||
251 | } | ||
252 | |||
253 | private getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { | ||
254 | return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` | ||
255 | } | ||
256 | } | ||
diff --git a/shared/extra-utils/ffprobe.ts b/shared/ffmpeg/ffprobe.ts index 7efc58a0d..fda08c28e 100644 --- a/shared/extra-utils/ffprobe.ts +++ b/shared/ffmpeg/ffprobe.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { ffprobe, FfprobeData } from 'fluent-ffmpeg' | 1 | import { ffprobe, FfprobeData } from 'fluent-ffmpeg' |
2 | import { forceNumber } from '@shared/core-utils' | 2 | import { forceNumber } from '@shared/core-utils' |
3 | import { VideoFileMetadata, VideoResolution } from '@shared/models/videos' | 3 | import { VideoResolution } from '@shared/models/videos' |
4 | 4 | ||
5 | /** | 5 | /** |
6 | * | 6 | * |
@@ -141,35 +141,29 @@ async function getVideoStreamFPS (path: string, existingProbe?: FfprobeData) { | |||
141 | return 0 | 141 | return 0 |
142 | } | 142 | } |
143 | 143 | ||
144 | async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { | ||
145 | const metadata = existingProbe || await ffprobePromise(path) | ||
146 | |||
147 | return new VideoFileMetadata(metadata) | ||
148 | } | ||
149 | |||
150 | async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise<number> { | 144 | async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise<number> { |
151 | const metadata = await buildFileMetadata(path, existingProbe) | 145 | const metadata = existingProbe || await ffprobePromise(path) |
152 | 146 | ||
153 | let bitrate = metadata.format.bit_rate as number | 147 | let bitrate = metadata.format.bit_rate |
154 | if (bitrate && !isNaN(bitrate)) return bitrate | 148 | if (bitrate && !isNaN(bitrate)) return bitrate |
155 | 149 | ||
156 | const videoStream = await getVideoStream(path, existingProbe) | 150 | const videoStream = await getVideoStream(path, existingProbe) |
157 | if (!videoStream) return undefined | 151 | if (!videoStream) return undefined |
158 | 152 | ||
159 | bitrate = videoStream?.bit_rate | 153 | bitrate = forceNumber(videoStream?.bit_rate) |
160 | if (bitrate && !isNaN(bitrate)) return bitrate | 154 | if (bitrate && !isNaN(bitrate)) return bitrate |
161 | 155 | ||
162 | return undefined | 156 | return undefined |
163 | } | 157 | } |
164 | 158 | ||
165 | async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) { | 159 | async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) { |
166 | const metadata = await buildFileMetadata(path, existingProbe) | 160 | const metadata = existingProbe || await ffprobePromise(path) |
167 | 161 | ||
168 | return Math.round(metadata.format.duration) | 162 | return Math.round(metadata.format.duration) |
169 | } | 163 | } |
170 | 164 | ||
171 | async function getVideoStream (path: string, existingProbe?: FfprobeData) { | 165 | async function getVideoStream (path: string, existingProbe?: FfprobeData) { |
172 | const metadata = await buildFileMetadata(path, existingProbe) | 166 | const metadata = existingProbe || await ffprobePromise(path) |
173 | 167 | ||
174 | return metadata.streams.find(s => s.codec_type === 'video') | 168 | return metadata.streams.find(s => s.codec_type === 'video') |
175 | } | 169 | } |
@@ -178,7 +172,6 @@ async function getVideoStream (path: string, existingProbe?: FfprobeData) { | |||
178 | 172 | ||
179 | export { | 173 | export { |
180 | getVideoStreamDimensionsInfo, | 174 | getVideoStreamDimensionsInfo, |
181 | buildFileMetadata, | ||
182 | getMaxAudioBitrate, | 175 | getMaxAudioBitrate, |
183 | getVideoStream, | 176 | getVideoStream, |
184 | getVideoStreamDuration, | 177 | getVideoStreamDuration, |
diff --git a/shared/ffmpeg/index.ts b/shared/ffmpeg/index.ts new file mode 100644 index 000000000..07a7d5402 --- /dev/null +++ b/shared/ffmpeg/index.ts | |||
@@ -0,0 +1,8 @@ | |||
1 | export * from './ffmpeg-command-wrapper' | ||
2 | export * from './ffmpeg-edition' | ||
3 | export * from './ffmpeg-images' | ||
4 | export * from './ffmpeg-live' | ||
5 | export * from './ffmpeg-utils' | ||
6 | export * from './ffmpeg-version' | ||
7 | export * from './ffmpeg-vod' | ||
8 | export * from './ffprobe' | ||
diff --git a/shared/ffmpeg/shared/encoder-options.ts b/shared/ffmpeg/shared/encoder-options.ts new file mode 100644 index 000000000..9692a6b02 --- /dev/null +++ b/shared/ffmpeg/shared/encoder-options.ts | |||
@@ -0,0 +1,39 @@ | |||
1 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
2 | import { EncoderOptions } from '@shared/models' | ||
3 | import { buildStreamSuffix } from '../ffmpeg-utils' | ||
4 | |||
5 | export function addDefaultEncoderGlobalParams (command: FfmpegCommand) { | ||
6 | // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 | ||
7 | command.outputOption('-max_muxing_queue_size 1024') | ||
8 | // strip all metadata | ||
9 | .outputOption('-map_metadata -1') | ||
10 | // allows import of source material with incompatible pixel formats (e.g. MJPEG video) | ||
11 | .outputOption('-pix_fmt yuv420p') | ||
12 | } | ||
13 | |||
14 | export function addDefaultEncoderParams (options: { | ||
15 | command: FfmpegCommand | ||
16 | encoder: 'libx264' | string | ||
17 | fps: number | ||
18 | |||
19 | streamNum?: number | ||
20 | }) { | ||
21 | const { command, encoder, fps, streamNum } = options | ||
22 | |||
23 | if (encoder === 'libx264') { | ||
24 | // 3.1 is the minimal resource allocation for our highest supported resolution | ||
25 | command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') | ||
26 | |||
27 | if (fps) { | ||
28 | // Keyframe interval of 2 seconds for faster seeking and resolution switching. | ||
29 | // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html | ||
30 | // https://superuser.com/a/908325 | ||
31 | command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) | ||
32 | } | ||
33 | } | ||
34 | } | ||
35 | |||
36 | export function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions) { | ||
37 | command.inputOptions(options.inputOptions ?? []) | ||
38 | .outputOptions(options.outputOptions ?? []) | ||
39 | } | ||
diff --git a/shared/ffmpeg/shared/index.ts b/shared/ffmpeg/shared/index.ts new file mode 100644 index 000000000..51de0316f --- /dev/null +++ b/shared/ffmpeg/shared/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './encoder-options' | ||
2 | export * from './presets' | ||
diff --git a/shared/ffmpeg/shared/presets.ts b/shared/ffmpeg/shared/presets.ts new file mode 100644 index 000000000..dcebdc1cf --- /dev/null +++ b/shared/ffmpeg/shared/presets.ts | |||
@@ -0,0 +1,93 @@ | |||
1 | import { pick } from '@shared/core-utils' | ||
2 | import { FFmpegCommandWrapper } from '../ffmpeg-command-wrapper' | ||
3 | import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '../ffprobe' | ||
4 | import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './encoder-options' | ||
5 | import { getScaleFilter, StreamType } from '../ffmpeg-utils' | ||
6 | |||
7 | export async function presetVOD (options: { | ||
8 | commandWrapper: FFmpegCommandWrapper | ||
9 | |||
10 | input: string | ||
11 | |||
12 | canCopyAudio: boolean | ||
13 | canCopyVideo: boolean | ||
14 | |||
15 | resolution: number | ||
16 | fps: number | ||
17 | |||
18 | scaleFilterValue?: string | ||
19 | }) { | ||
20 | const { commandWrapper, input, resolution, fps, scaleFilterValue } = options | ||
21 | const command = commandWrapper.getCommand() | ||
22 | |||
23 | command.format('mp4') | ||
24 | .outputOption('-movflags faststart') | ||
25 | |||
26 | addDefaultEncoderGlobalParams(command) | ||
27 | |||
28 | const probe = await ffprobePromise(input) | ||
29 | |||
30 | // Audio encoder | ||
31 | const bitrate = await getVideoStreamBitrate(input, probe) | ||
32 | const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) | ||
33 | |||
34 | let streamsToProcess: StreamType[] = [ 'audio', 'video' ] | ||
35 | |||
36 | if (!await hasAudioStream(input, probe)) { | ||
37 | command.noAudio() | ||
38 | streamsToProcess = [ 'video' ] | ||
39 | } | ||
40 | |||
41 | for (const streamType of streamsToProcess) { | ||
42 | const builderResult = await commandWrapper.getEncoderBuilderResult({ | ||
43 | ...pick(options, [ 'canCopyAudio', 'canCopyVideo' ]), | ||
44 | |||
45 | input, | ||
46 | inputBitrate: bitrate, | ||
47 | inputRatio: videoStreamDimensions?.ratio || 0, | ||
48 | |||
49 | resolution, | ||
50 | fps, | ||
51 | streamType, | ||
52 | |||
53 | videoType: 'vod' as 'vod' | ||
54 | }) | ||
55 | |||
56 | if (!builderResult) { | ||
57 | throw new Error('No available encoder found for stream ' + streamType) | ||
58 | } | ||
59 | |||
60 | commandWrapper.debugLog( | ||
61 | `Apply ffmpeg params from ${builderResult.encoder} for ${streamType} ` + | ||
62 | `stream of input ${input} using ${commandWrapper.getProfile()} profile.`, | ||
63 | { builderResult, resolution, fps } | ||
64 | ) | ||
65 | |||
66 | if (streamType === 'video') { | ||
67 | command.videoCodec(builderResult.encoder) | ||
68 | |||
69 | if (scaleFilterValue) { | ||
70 | command.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) | ||
71 | } | ||
72 | } else if (streamType === 'audio') { | ||
73 | command.audioCodec(builderResult.encoder) | ||
74 | } | ||
75 | |||
76 | applyEncoderOptions(command, builderResult.result) | ||
77 | addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps }) | ||
78 | } | ||
79 | } | ||
80 | |||
81 | export function presetCopy (commandWrapper: FFmpegCommandWrapper) { | ||
82 | commandWrapper.getCommand() | ||
83 | .format('mp4') | ||
84 | .videoCodec('copy') | ||
85 | .audioCodec('copy') | ||
86 | } | ||
87 | |||
88 | export function presetOnlyAudio (commandWrapper: FFmpegCommandWrapper) { | ||
89 | commandWrapper.getCommand() | ||
90 | .format('mp4') | ||
91 | .audioCodec('copy') | ||
92 | .noVideo() | ||
93 | } | ||
diff --git a/shared/models/index.ts b/shared/models/index.ts index 439e9c8e1..78f6e73e3 100644 --- a/shared/models/index.ts +++ b/shared/models/index.ts | |||
@@ -11,6 +11,7 @@ export * from './moderation' | |||
11 | export * from './overviews' | 11 | export * from './overviews' |
12 | export * from './plugins' | 12 | export * from './plugins' |
13 | export * from './redundancy' | 13 | export * from './redundancy' |
14 | export * from './runners' | ||
14 | export * from './search' | 15 | export * from './search' |
15 | export * from './server' | 16 | export * from './server' |
16 | export * from './tokens' | 17 | export * from './tokens' |
diff --git a/shared/models/runners/abort-runner-job-body.model.ts b/shared/models/runners/abort-runner-job-body.model.ts new file mode 100644 index 000000000..0b9c46c91 --- /dev/null +++ b/shared/models/runners/abort-runner-job-body.model.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export interface AbortRunnerJobBody { | ||
2 | runnerToken: string | ||
3 | jobToken: string | ||
4 | |||
5 | reason: string | ||
6 | } | ||
diff --git a/shared/models/runners/accept-runner-job-body.model.ts b/shared/models/runners/accept-runner-job-body.model.ts new file mode 100644 index 000000000..cb266c4e6 --- /dev/null +++ b/shared/models/runners/accept-runner-job-body.model.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export interface AcceptRunnerJobBody { | ||
2 | runnerToken: string | ||
3 | } | ||
diff --git a/shared/models/runners/accept-runner-job-result.model.ts b/shared/models/runners/accept-runner-job-result.model.ts new file mode 100644 index 000000000..f2094b945 --- /dev/null +++ b/shared/models/runners/accept-runner-job-result.model.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | import { RunnerJobPayload } from './runner-job-payload.model' | ||
2 | import { RunnerJob } from './runner-job.model' | ||
3 | |||
4 | export interface AcceptRunnerJobResult <T extends RunnerJobPayload = RunnerJobPayload> { | ||
5 | job: RunnerJob<T> & { jobToken: string } | ||
6 | } | ||
diff --git a/shared/models/runners/error-runner-job-body.model.ts b/shared/models/runners/error-runner-job-body.model.ts new file mode 100644 index 000000000..ac8568409 --- /dev/null +++ b/shared/models/runners/error-runner-job-body.model.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export interface ErrorRunnerJobBody { | ||
2 | runnerToken: string | ||
3 | jobToken: string | ||
4 | |||
5 | message: string | ||
6 | } | ||
diff --git a/shared/models/runners/index.ts b/shared/models/runners/index.ts new file mode 100644 index 000000000..a52b82d2e --- /dev/null +++ b/shared/models/runners/index.ts | |||
@@ -0,0 +1,21 @@ | |||
1 | export * from './abort-runner-job-body.model' | ||
2 | export * from './accept-runner-job-body.model' | ||
3 | export * from './accept-runner-job-result.model' | ||
4 | export * from './error-runner-job-body.model' | ||
5 | export * from './list-runner-jobs-query.model' | ||
6 | export * from './list-runner-registration-tokens.model' | ||
7 | export * from './list-runners-query.model' | ||
8 | export * from './register-runner-body.model' | ||
9 | export * from './register-runner-result.model' | ||
10 | export * from './request-runner-job-body.model' | ||
11 | export * from './request-runner-job-result.model' | ||
12 | export * from './runner-job-payload.model' | ||
13 | export * from './runner-job-private-payload.model' | ||
14 | export * from './runner-job-state.model' | ||
15 | export * from './runner-job-success-body.model' | ||
16 | export * from './runner-job-type.type' | ||
17 | export * from './runner-job-update-body.model' | ||
18 | export * from './runner-job.model' | ||
19 | export * from './runner-registration-token' | ||
20 | export * from './runner.model' | ||
21 | export * from './unregister-runner-body.model' | ||
diff --git a/shared/models/runners/list-runner-jobs-query.model.ts b/shared/models/runners/list-runner-jobs-query.model.ts new file mode 100644 index 000000000..a5b62c55d --- /dev/null +++ b/shared/models/runners/list-runner-jobs-query.model.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export interface ListRunnerJobsQuery { | ||
2 | start?: number | ||
3 | count?: number | ||
4 | sort?: string | ||
5 | search?: string | ||
6 | } | ||
diff --git a/shared/models/runners/list-runner-registration-tokens.model.ts b/shared/models/runners/list-runner-registration-tokens.model.ts new file mode 100644 index 000000000..872e059cf --- /dev/null +++ b/shared/models/runners/list-runner-registration-tokens.model.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | export interface ListRunnerRegistrationTokensQuery { | ||
2 | start?: number | ||
3 | count?: number | ||
4 | sort?: string | ||
5 | } | ||
diff --git a/shared/models/runners/list-runners-query.model.ts b/shared/models/runners/list-runners-query.model.ts new file mode 100644 index 000000000..d4362e4c5 --- /dev/null +++ b/shared/models/runners/list-runners-query.model.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | export interface ListRunnersQuery { | ||
2 | start?: number | ||
3 | count?: number | ||
4 | sort?: string | ||
5 | } | ||
diff --git a/shared/models/runners/register-runner-body.model.ts b/shared/models/runners/register-runner-body.model.ts new file mode 100644 index 000000000..969bb35e1 --- /dev/null +++ b/shared/models/runners/register-runner-body.model.ts | |||
@@ -0,0 +1,6 @@ | |||
1 | export interface RegisterRunnerBody { | ||
2 | registrationToken: string | ||
3 | |||
4 | name: string | ||
5 | description?: string | ||
6 | } | ||
diff --git a/shared/models/runners/register-runner-result.model.ts b/shared/models/runners/register-runner-result.model.ts new file mode 100644 index 000000000..e31776c6a --- /dev/null +++ b/shared/models/runners/register-runner-result.model.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export interface RegisterRunnerResult { | ||
2 | id: number | ||
3 | runnerToken: string | ||
4 | } | ||
diff --git a/shared/models/runners/request-runner-job-body.model.ts b/shared/models/runners/request-runner-job-body.model.ts new file mode 100644 index 000000000..0970d9007 --- /dev/null +++ b/shared/models/runners/request-runner-job-body.model.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export interface RequestRunnerJobBody { | ||
2 | runnerToken: string | ||
3 | } | ||
diff --git a/shared/models/runners/request-runner-job-result.model.ts b/shared/models/runners/request-runner-job-result.model.ts new file mode 100644 index 000000000..98601c42c --- /dev/null +++ b/shared/models/runners/request-runner-job-result.model.ts | |||
@@ -0,0 +1,10 @@ | |||
1 | import { RunnerJobPayload } from './runner-job-payload.model' | ||
2 | import { RunnerJobType } from './runner-job-type.type' | ||
3 | |||
4 | export interface RequestRunnerJobResult <P extends RunnerJobPayload = RunnerJobPayload> { | ||
5 | availableJobs: { | ||
6 | uuid: string | ||
7 | type: RunnerJobType | ||
8 | payload: P | ||
9 | }[] | ||
10 | } | ||
diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts new file mode 100644 index 000000000..8f0c17135 --- /dev/null +++ b/shared/models/runners/runner-job-payload.model.ts | |||
@@ -0,0 +1,68 @@ | |||
1 | export type RunnerJobVODPayload = | ||
2 | RunnerJobVODWebVideoTranscodingPayload | | ||
3 | RunnerJobVODHLSTranscodingPayload | | ||
4 | RunnerJobVODAudioMergeTranscodingPayload | ||
5 | |||
6 | export type RunnerJobPayload = | ||
7 | RunnerJobVODPayload | | ||
8 | RunnerJobLiveRTMPHLSTranscodingPayload | ||
9 | |||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
12 | export interface RunnerJobVODWebVideoTranscodingPayload { | ||
13 | input: { | ||
14 | videoFileUrl: string | ||
15 | } | ||
16 | |||
17 | output: { | ||
18 | resolution: number | ||
19 | fps: number | ||
20 | } | ||
21 | } | ||
22 | |||
23 | export interface RunnerJobVODHLSTranscodingPayload { | ||
24 | input: { | ||
25 | videoFileUrl: string | ||
26 | } | ||
27 | |||
28 | output: { | ||
29 | resolution: number | ||
30 | fps: number | ||
31 | } | ||
32 | } | ||
33 | |||
34 | export interface RunnerJobVODAudioMergeTranscodingPayload { | ||
35 | input: { | ||
36 | audioFileUrl: string | ||
37 | previewFileUrl: string | ||
38 | } | ||
39 | |||
40 | output: { | ||
41 | resolution: number | ||
42 | fps: number | ||
43 | } | ||
44 | } | ||
45 | |||
46 | // --------------------------------------------------------------------------- | ||
47 | |||
48 | export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { | ||
49 | return !!(payload as RunnerJobVODAudioMergeTranscodingPayload).input.audioFileUrl | ||
50 | } | ||
51 | |||
52 | // --------------------------------------------------------------------------- | ||
53 | |||
54 | export interface RunnerJobLiveRTMPHLSTranscodingPayload { | ||
55 | input: { | ||
56 | rtmpUrl: string | ||
57 | } | ||
58 | |||
59 | output: { | ||
60 | toTranscode: { | ||
61 | resolution: number | ||
62 | fps: number | ||
63 | }[] | ||
64 | |||
65 | segmentDuration: number | ||
66 | segmentListSize: number | ||
67 | } | ||
68 | } | ||
diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts new file mode 100644 index 000000000..c1d8d1045 --- /dev/null +++ b/shared/models/runners/runner-job-private-payload.model.ts | |||
@@ -0,0 +1,34 @@ | |||
1 | export type RunnerJobVODPrivatePayload = | ||
2 | RunnerJobVODWebVideoTranscodingPrivatePayload | | ||
3 | RunnerJobVODAudioMergeTranscodingPrivatePayload | | ||
4 | RunnerJobVODHLSTranscodingPrivatePayload | ||
5 | |||
6 | export type RunnerJobPrivatePayload = | ||
7 | RunnerJobVODPrivatePayload | | ||
8 | RunnerJobLiveRTMPHLSTranscodingPrivatePayload | ||
9 | |||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
12 | export interface RunnerJobVODWebVideoTranscodingPrivatePayload { | ||
13 | videoUUID: string | ||
14 | isNewVideo: boolean | ||
15 | } | ||
16 | |||
17 | export interface RunnerJobVODAudioMergeTranscodingPrivatePayload { | ||
18 | videoUUID: string | ||
19 | isNewVideo: boolean | ||
20 | } | ||
21 | |||
22 | export interface RunnerJobVODHLSTranscodingPrivatePayload { | ||
23 | videoUUID: string | ||
24 | isNewVideo: boolean | ||
25 | deleteWebVideoFiles: boolean | ||
26 | } | ||
27 | |||
28 | // --------------------------------------------------------------------------- | ||
29 | |||
30 | export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload { | ||
31 | videoUUID: string | ||
32 | masterPlaylistName: string | ||
33 | outputDirectory: string | ||
34 | } | ||
diff --git a/shared/models/runners/runner-job-state.model.ts b/shared/models/runners/runner-job-state.model.ts new file mode 100644 index 000000000..738db38b7 --- /dev/null +++ b/shared/models/runners/runner-job-state.model.ts | |||
@@ -0,0 +1,10 @@ | |||
1 | export enum RunnerJobState { | ||
2 | PENDING = 1, | ||
3 | PROCESSING = 2, | ||
4 | COMPLETED = 3, | ||
5 | ERRORED = 4, | ||
6 | WAITING_FOR_PARENT_JOB = 5, | ||
7 | CANCELLED = 6, | ||
8 | PARENT_ERRORED = 7, | ||
9 | PARENT_CANCELLED = 8 | ||
10 | } | ||
diff --git a/shared/models/runners/runner-job-success-body.model.ts b/shared/models/runners/runner-job-success-body.model.ts new file mode 100644 index 000000000..223b7552d --- /dev/null +++ b/shared/models/runners/runner-job-success-body.model.ts | |||
@@ -0,0 +1,41 @@ | |||
1 | export interface RunnerJobSuccessBody { | ||
2 | runnerToken: string | ||
3 | jobToken: string | ||
4 | |||
5 | payload: RunnerJobSuccessPayload | ||
6 | } | ||
7 | |||
8 | // --------------------------------------------------------------------------- | ||
9 | |||
10 | export type RunnerJobSuccessPayload = | ||
11 | VODWebVideoTranscodingSuccess | | ||
12 | VODHLSTranscodingSuccess | | ||
13 | VODAudioMergeTranscodingSuccess | | ||
14 | LiveRTMPHLSTranscodingSuccess | ||
15 | |||
16 | export interface VODWebVideoTranscodingSuccess { | ||
17 | videoFile: Blob | string | ||
18 | } | ||
19 | |||
20 | export interface VODHLSTranscodingSuccess { | ||
21 | videoFile: Blob | string | ||
22 | resolutionPlaylistFile: Blob | string | ||
23 | } | ||
24 | |||
25 | export interface VODAudioMergeTranscodingSuccess { | ||
26 | videoFile: Blob | string | ||
27 | } | ||
28 | |||
29 | export interface LiveRTMPHLSTranscodingSuccess { | ||
30 | |||
31 | } | ||
32 | |||
33 | export function isWebVideoOrAudioMergeTranscodingPayloadSuccess ( | ||
34 | payload: RunnerJobSuccessPayload | ||
35 | ): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess { | ||
36 | return !!(payload as VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess)?.videoFile | ||
37 | } | ||
38 | |||
39 | export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload): payload is VODHLSTranscodingSuccess { | ||
40 | return !!(payload as VODHLSTranscodingSuccess)?.resolutionPlaylistFile | ||
41 | } | ||
diff --git a/shared/models/runners/runner-job-type.type.ts b/shared/models/runners/runner-job-type.type.ts new file mode 100644 index 000000000..36d3b9b25 --- /dev/null +++ b/shared/models/runners/runner-job-type.type.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | export type RunnerJobType = | ||
2 | 'vod-web-video-transcoding' | | ||
3 | 'vod-hls-transcoding' | | ||
4 | 'vod-audio-merge-transcoding' | | ||
5 | 'live-rtmp-hls-transcoding' | ||
diff --git a/shared/models/runners/runner-job-update-body.model.ts b/shared/models/runners/runner-job-update-body.model.ts new file mode 100644 index 000000000..ed94bbe63 --- /dev/null +++ b/shared/models/runners/runner-job-update-body.model.ts | |||
@@ -0,0 +1,28 @@ | |||
1 | export interface RunnerJobUpdateBody { | ||
2 | runnerToken: string | ||
3 | jobToken: string | ||
4 | |||
5 | progress?: number | ||
6 | payload?: RunnerJobUpdatePayload | ||
7 | } | ||
8 | |||
9 | // --------------------------------------------------------------------------- | ||
10 | |||
11 | export type RunnerJobUpdatePayload = LiveRTMPHLSTranscodingUpdatePayload | ||
12 | |||
13 | export interface LiveRTMPHLSTranscodingUpdatePayload { | ||
14 | type: 'add-chunk' | 'remove-chunk' | ||
15 | |||
16 | masterPlaylistFile?: Blob | string | ||
17 | |||
18 | resolutionPlaylistFilename?: string | ||
19 | resolutionPlaylistFile?: Blob | string | ||
20 | |||
21 | videoChunkFilename: string | ||
22 | videoChunkFile?: Blob | string | ||
23 | } | ||
24 | |||
25 | export function isLiveRTMPHLSTranscodingUpdatePayload (value: RunnerJobUpdatePayload): value is LiveRTMPHLSTranscodingUpdatePayload { | ||
26 | // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion | ||
27 | return !!(value as LiveRTMPHLSTranscodingUpdatePayload)?.videoChunkFilename | ||
28 | } | ||
diff --git a/shared/models/runners/runner-job.model.ts b/shared/models/runners/runner-job.model.ts new file mode 100644 index 000000000..080093563 --- /dev/null +++ b/shared/models/runners/runner-job.model.ts | |||
@@ -0,0 +1,45 @@ | |||
1 | import { VideoConstant } from '../videos' | ||
2 | import { RunnerJobPayload } from './runner-job-payload.model' | ||
3 | import { RunnerJobPrivatePayload } from './runner-job-private-payload.model' | ||
4 | import { RunnerJobState } from './runner-job-state.model' | ||
5 | import { RunnerJobType } from './runner-job-type.type' | ||
6 | |||
7 | export interface RunnerJob <T extends RunnerJobPayload = RunnerJobPayload> { | ||
8 | uuid: string | ||
9 | |||
10 | type: RunnerJobType | ||
11 | |||
12 | state: VideoConstant<RunnerJobState> | ||
13 | |||
14 | payload: T | ||
15 | |||
16 | failures: number | ||
17 | error: string | null | ||
18 | |||
19 | progress: number | ||
20 | priority: number | ||
21 | |||
22 | startedAt: Date | string | ||
23 | createdAt: Date | string | ||
24 | updatedAt: Date | string | ||
25 | finishedAt: Date | string | ||
26 | |||
27 | parent?: { | ||
28 | type: RunnerJobType | ||
29 | state: VideoConstant<RunnerJobState> | ||
30 | uuid: string | ||
31 | } | ||
32 | |||
33 | // If associated to a runner | ||
34 | runner?: { | ||
35 | id: number | ||
36 | name: string | ||
37 | |||
38 | description: string | ||
39 | } | ||
40 | } | ||
41 | |||
42 | // eslint-disable-next-line max-len | ||
43 | export interface RunnerJobAdmin <T extends RunnerJobPayload = RunnerJobPayload, U extends RunnerJobPrivatePayload = RunnerJobPrivatePayload> extends RunnerJob<T> { | ||
44 | privatePayload: U | ||
45 | } | ||
diff --git a/shared/models/runners/runner-registration-token.ts b/shared/models/runners/runner-registration-token.ts new file mode 100644 index 000000000..0a157aa51 --- /dev/null +++ b/shared/models/runners/runner-registration-token.ts | |||
@@ -0,0 +1,10 @@ | |||
1 | export interface RunnerRegistrationToken { | ||
2 | id: number | ||
3 | |||
4 | registrationToken: string | ||
5 | |||
6 | createdAt: Date | ||
7 | updatedAt: Date | ||
8 | |||
9 | registeredRunnersCount: number | ||
10 | } | ||
diff --git a/shared/models/runners/runner.model.ts b/shared/models/runners/runner.model.ts new file mode 100644 index 000000000..3284f2992 --- /dev/null +++ b/shared/models/runners/runner.model.ts | |||
@@ -0,0 +1,12 @@ | |||
1 | export interface Runner { | ||
2 | id: number | ||
3 | |||
4 | name: string | ||
5 | description: string | ||
6 | |||
7 | ip: string | ||
8 | lastContact: Date | string | ||
9 | |||
10 | createdAt: Date | string | ||
11 | updatedAt: Date | string | ||
12 | } | ||
diff --git a/shared/models/runners/unregister-runner-body.model.ts b/shared/models/runners/unregister-runner-body.model.ts new file mode 100644 index 000000000..d3465c5d6 --- /dev/null +++ b/shared/models/runners/unregister-runner-body.model.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | export interface UnregisterRunnerBody { | ||
2 | runnerToken: string | ||
3 | } | ||
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 6ffe3a676..5d2c10278 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts | |||
@@ -116,6 +116,10 @@ export interface CustomConfig { | |||
116 | allowAdditionalExtensions: boolean | 116 | allowAdditionalExtensions: boolean |
117 | allowAudioFiles: boolean | 117 | allowAudioFiles: boolean |
118 | 118 | ||
119 | remoteRunners: { | ||
120 | enabled: boolean | ||
121 | } | ||
122 | |||
119 | threads: number | 123 | threads: number |
120 | concurrency: number | 124 | concurrency: number |
121 | 125 | ||
@@ -149,6 +153,9 @@ export interface CustomConfig { | |||
149 | 153 | ||
150 | transcoding: { | 154 | transcoding: { |
151 | enabled: boolean | 155 | enabled: boolean |
156 | remoteRunners: { | ||
157 | enabled: boolean | ||
158 | } | ||
152 | threads: number | 159 | threads: number |
153 | profile: string | 160 | profile: string |
154 | resolutions: ConfigResolutions | 161 | resolutions: ConfigResolutions |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 9c0b5ea56..16187d133 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -18,6 +18,7 @@ export type JobType = | |||
18 | | 'after-video-channel-import' | 18 | | 'after-video-channel-import' |
19 | | 'email' | 19 | | 'email' |
20 | | 'federate-video' | 20 | | 'federate-video' |
21 | | 'transcoding-job-builder' | ||
21 | | 'manage-video-torrent' | 22 | | 'manage-video-torrent' |
22 | | 'move-to-object-storage' | 23 | | 'move-to-object-storage' |
23 | | 'notify' | 24 | | 'notify' |
@@ -41,6 +42,10 @@ export interface Job { | |||
41 | createdAt: Date | string | 42 | createdAt: Date | string |
42 | finishedOn: Date | string | 43 | finishedOn: Date | string |
43 | processedOn: Date | string | 44 | processedOn: Date | string |
45 | |||
46 | parent?: { | ||
47 | id: string | ||
48 | } | ||
44 | } | 49 | } |
45 | 50 | ||
46 | export type ActivitypubHttpBroadcastPayload = { | 51 | export type ActivitypubHttpBroadcastPayload = { |
@@ -139,30 +144,28 @@ interface BaseTranscodingPayload { | |||
139 | export interface HLSTranscodingPayload extends BaseTranscodingPayload { | 144 | export interface HLSTranscodingPayload extends BaseTranscodingPayload { |
140 | type: 'new-resolution-to-hls' | 145 | type: 'new-resolution-to-hls' |
141 | resolution: VideoResolution | 146 | resolution: VideoResolution |
147 | fps: number | ||
142 | copyCodecs: boolean | 148 | copyCodecs: boolean |
143 | 149 | ||
144 | hasAudio: boolean | 150 | deleteWebTorrentFiles: boolean |
145 | |||
146 | autoDeleteWebTorrentIfNeeded: boolean | ||
147 | isMaxQuality: boolean | ||
148 | } | 151 | } |
149 | 152 | ||
150 | export interface NewWebTorrentResolutionTranscodingPayload extends BaseTranscodingPayload { | 153 | export interface NewWebTorrentResolutionTranscodingPayload extends BaseTranscodingPayload { |
151 | type: 'new-resolution-to-webtorrent' | 154 | type: 'new-resolution-to-webtorrent' |
152 | resolution: VideoResolution | 155 | resolution: VideoResolution |
153 | 156 | fps: number | |
154 | hasAudio: boolean | ||
155 | createHLSIfNeeded: boolean | ||
156 | } | 157 | } |
157 | 158 | ||
158 | export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload { | 159 | export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload { |
159 | type: 'merge-audio-to-webtorrent' | 160 | type: 'merge-audio-to-webtorrent' |
160 | resolution: VideoResolution | 161 | resolution: VideoResolution |
161 | createHLSIfNeeded: true | 162 | fps: number |
162 | } | 163 | } |
163 | 164 | ||
164 | export interface OptimizeTranscodingPayload extends BaseTranscodingPayload { | 165 | export interface OptimizeTranscodingPayload extends BaseTranscodingPayload { |
165 | type: 'optimize-to-webtorrent' | 166 | type: 'optimize-to-webtorrent' |
167 | |||
168 | quickTranscode: boolean | ||
166 | } | 169 | } |
167 | 170 | ||
168 | export type VideoTranscodingPayload = | 171 | export type VideoTranscodingPayload = |
@@ -258,3 +261,27 @@ export interface FederateVideoPayload { | |||
258 | videoUUID: string | 261 | videoUUID: string |
259 | isNewVideo: boolean | 262 | isNewVideo: boolean |
260 | } | 263 | } |
264 | |||
265 | // --------------------------------------------------------------------------- | ||
266 | |||
267 | export interface TranscodingJobBuilderPayload { | ||
268 | videoUUID: string | ||
269 | |||
270 | optimizeJob?: { | ||
271 | isNewVideo: boolean | ||
272 | } | ||
273 | |||
274 | // Array of jobs to create | ||
275 | jobs?: { | ||
276 | type: 'video-transcoding' | ||
277 | payload: VideoTranscodingPayload | ||
278 | priority?: number | ||
279 | }[] | ||
280 | |||
281 | // Array of sequential jobs to create | ||
282 | sequentialJobs?: { | ||
283 | type: 'video-transcoding' | ||
284 | payload: VideoTranscodingPayload | ||
285 | priority?: number | ||
286 | }[][] | ||
287 | } | ||
diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index d0bd9a00f..38b9d0385 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts | |||
@@ -148,6 +148,10 @@ export interface ServerConfig { | |||
148 | 148 | ||
149 | profile: string | 149 | profile: string |
150 | availableProfiles: string[] | 150 | availableProfiles: string[] |
151 | |||
152 | remoteRunners: { | ||
153 | enabled: boolean | ||
154 | } | ||
151 | } | 155 | } |
152 | 156 | ||
153 | live: { | 157 | live: { |
@@ -165,6 +169,10 @@ export interface ServerConfig { | |||
165 | transcoding: { | 169 | transcoding: { |
166 | enabled: boolean | 170 | enabled: boolean |
167 | 171 | ||
172 | remoteRunners: { | ||
173 | enabled: boolean | ||
174 | } | ||
175 | |||
168 | enabledResolutions: number[] | 176 | enabledResolutions: number[] |
169 | 177 | ||
170 | profile: string | 178 | profile: string |
diff --git a/shared/models/server/server-error-code.enum.ts b/shared/models/server/server-error-code.enum.ts index a39cde1b3..24d3c6d21 100644 --- a/shared/models/server/server-error-code.enum.ts +++ b/shared/models/server/server-error-code.enum.ts | |||
@@ -45,7 +45,10 @@ export const enum ServerErrorCode { | |||
45 | INVALID_TWO_FACTOR = 'invalid_two_factor', | 45 | INVALID_TWO_FACTOR = 'invalid_two_factor', |
46 | 46 | ||
47 | ACCOUNT_WAITING_FOR_APPROVAL = 'account_waiting_for_approval', | 47 | ACCOUNT_WAITING_FOR_APPROVAL = 'account_waiting_for_approval', |
48 | ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected' | 48 | ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected', |
49 | |||
50 | RUNNER_JOB_NOT_IN_PROCESSING_STATE = 'runner_job_not_in_processing_state', | ||
51 | UNKNOWN_RUNNER_TOKEN = 'unknown_runner_token' | ||
49 | } | 52 | } |
50 | 53 | ||
51 | /** | 54 | /** |
diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index 42e5c8cd6..a5a770b75 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts | |||
@@ -45,5 +45,7 @@ export const enum UserRight { | |||
45 | 45 | ||
46 | MANAGE_VIDEO_IMPORTS = 27, | 46 | MANAGE_VIDEO_IMPORTS = 27, |
47 | 47 | ||
48 | MANAGE_REGISTRATIONS = 28 | 48 | MANAGE_REGISTRATIONS = 28, |
49 | |||
50 | MANAGE_RUNNERS = 29 | ||
49 | } | 51 | } |
diff --git a/shared/models/videos/live/live-video-error.enum.ts b/shared/models/videos/live/live-video-error.enum.ts index 3a8e4afa0..a26453505 100644 --- a/shared/models/videos/live/live-video-error.enum.ts +++ b/shared/models/videos/live/live-video-error.enum.ts | |||
@@ -3,5 +3,7 @@ export const enum LiveVideoError { | |||
3 | DURATION_EXCEEDED = 2, | 3 | DURATION_EXCEEDED = 2, |
4 | QUOTA_EXCEEDED = 3, | 4 | QUOTA_EXCEEDED = 3, |
5 | FFMPEG_ERROR = 4, | 5 | FFMPEG_ERROR = 4, |
6 | BLACKLISTED = 5 | 6 | BLACKLISTED = 5, |
7 | RUNNER_JOB_ERROR = 6, | ||
8 | RUNNER_JOB_CANCEL = 7 | ||
7 | } | 9 | } |