aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-01-21 14:42:43 +0100
committerChocobozzz <me@florianbigard.com>2021-01-21 14:42:43 +0100
commit3b01f4c0ac764ecb70efaadfd939ca868c28769c (patch)
tree99c0cdef6dac0d43dd16c02b8bae3c132037cda6
parentd44cdcd766fbccbe4b96f34c11a64f0e2168c3b9 (diff)
downloadPeerTube-3b01f4c0ac764ecb70efaadfd939ca868c28769c.tar.gz
PeerTube-3b01f4c0ac764ecb70efaadfd939ca868c28769c.tar.zst
PeerTube-3b01f4c0ac764ecb70efaadfd939ca868c28769c.zip
Support progress for ffmpeg tasks
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.html13
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.scss3
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.ts10
-rw-r--r--server/controllers/api/jobs.ts13
-rw-r--r--server/helpers/ffmpeg-utils.ts20
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts9
-rw-r--r--server/lib/video-transcoding.ts36
-rw-r--r--shared/models/server/job.model.ts1
8 files changed, 71 insertions, 34 deletions
diff --git a/client/src/app/+admin/system/jobs/jobs.component.html b/client/src/app/+admin/system/jobs/jobs.component.html
index 2d60e7b9e..b6457a005 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.html
+++ b/client/src/app/+admin/system/jobs/jobs.component.html
@@ -40,7 +40,8 @@
40 <th style="width: 40px"></th> 40 <th style="width: 40px"></th>
41 <th style="width: calc(100% - 390px)" class="job-id" i18n>ID</th> 41 <th style="width: calc(100% - 390px)" class="job-id" i18n>ID</th>
42 <th style="width: 200px" class="job-type" i18n>Type</th> 42 <th style="width: 200px" class="job-type" i18n>Type</th>
43 <th style="width: 200px" class="job-type" i18n *ngIf="jobState === 'all'">State</th> 43 <th style="width: 200px" class="job-state" i18n *ngIf="jobState === 'all'">State</th>
44 <th style="width: 100px" class="job-progress" i18n *ngIf="hasProgress()">Progress</th>
44 <th style="width: 150px" class="job-date" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th> 45 <th style="width: 150px" class="job-date" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
45 </tr> 46 </tr>
46 </ng-template> 47 </ng-template>
@@ -55,9 +56,15 @@
55 56
56 <td class="job-id c-hand" [pRowToggler]="job" [title]="job.id">{{ job.id }}</td> 57 <td class="job-id c-hand" [pRowToggler]="job" [title]="job.id">{{ job.id }}</td>
57 <td class="job-type c-hand" [pRowToggler]="job">{{ job.type }}</td> 58 <td class="job-type c-hand" [pRowToggler]="job">{{ job.type }}</td>
58 <td class="job-type c-hand" [pRowToggler]="job" *ngIf="jobState === 'all'"> 59
60 <td class="job-state c-hand" [pRowToggler]="job" *ngIf="jobState === 'all'">
59 <span class="badge" [ngClass]="getJobStateClass(job.state)">{{ job.state }}</span> 61 <span class="badge" [ngClass]="getJobStateClass(job.state)">{{ job.state }}</span>
60 </td> 62 </td>
63
64 <td class="job-state" [pRowToggler]="job" *ngIf="hasProgress()">
65 {{ getProgress(job) }}
66 </td>
67
61 <td class="job-date c-hand" [pRowToggler]="job">{{ job.createdAt | date: 'short' }}</td> 68 <td class="job-date c-hand" [pRowToggler]="job">{{ job.createdAt | date: 'short' }}</td>
62 </tr> 69 </tr>
63 </ng-template> 70 </ng-template>
@@ -94,7 +101,7 @@
94 <ng-container *ngIf="jobType === 'all'" i18n>No jobs found.</ng-container> 101 <ng-container *ngIf="jobType === 'all'" i18n>No jobs found.</ng-container>
95 <ng-container *ngIf="jobType !== 'all'" i18n>No <code>{{ jobType }}</code> jobs found.</ng-container> 102 <ng-container *ngIf="jobType !== 'all'" i18n>No <code>{{ jobType }}</code> jobs found.</ng-container>
96 </ng-container> 103 </ng-container>
97 <ng-container *ngIf="jobState !== 'all'"> 104 <ng-container *ngIf="jobState !== 'all'">
98 <ng-container *ngIf="jobType === 'all'" i18n>No <span class="badge" [ngClass]="getJobStateClass(jobState)">{{ jobState }}</span> jobs found.</ng-container> 105 <ng-container *ngIf="jobType === 'all'" i18n>No <span class="badge" [ngClass]="getJobStateClass(jobState)">{{ jobState }}</span> jobs found.</ng-container>
99 <ng-container *ngIf="jobType !== 'all'" i18n>No <code>{{ jobType }}</code> jobs found that are <span class="badge" [ngClass]="getJobStateClass(jobState)">{{ jobState }}</span>.</ng-container> 106 <ng-container *ngIf="jobType !== 'all'" i18n>No <code>{{ jobType }}</code> jobs found that are <span class="badge" [ngClass]="getJobStateClass(jobState)">{{ jobState }}</span>.</ng-container>
100 </ng-container> 107 </ng-container>
diff --git a/client/src/app/+admin/system/jobs/jobs.component.scss b/client/src/app/+admin/system/jobs/jobs.component.scss
index 784ec4572..9c6ae73e1 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.scss
+++ b/client/src/app/+admin/system/jobs/jobs.component.scss
@@ -9,7 +9,8 @@
9 max-width: 30vw !important; 9 max-width: 30vw !important;
10} 10}
11 11
12.job-type { 12.job-type,
13.job-state {
13 width: 150px !important; 14 width: 150px !important;
14} 15}
15 16
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index b1940b0d3..6ab17b3c1 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -83,6 +83,16 @@ export class JobsComponent extends RestTable implements OnInit {
83 this.saveJobStateAndType() 83 this.saveJobStateAndType()
84 } 84 }
85 85
86 hasProgress () {
87 return this.jobType === 'all' || this.jobType === 'video-transcoding'
88 }
89
90 getProgress (job: Job) {
91 if (job.state === 'active') return job.progress + '%'
92
93 return ''
94 }
95
86 protected loadData () { 96 protected loadData () {
87 let jobState = this.jobState as JobState 97 let jobState = this.jobState as JobState
88 if (this.jobState === 'all') jobState = null 98 if (this.jobState === 'all') jobState = null
diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts
index e14ea2575..929140140 100644
--- a/server/controllers/api/jobs.ts
+++ b/server/controllers/api/jobs.ts
@@ -52,28 +52,23 @@ async function listJobs (req: express.Request, res: express.Response) {
52 52
53 const result: ResultList<Job> = { 53 const result: ResultList<Job> = {
54 total, 54 total,
55 data: state 55 data: await Promise.all(jobs.map(j => formatJob(j, state)))
56 ? jobs.map(j => formatJob(j, state))
57 : await Promise.all(jobs.map(j => formatJobWithUnknownState(j)))
58 } 56 }
59 57
60 return res.json(result) 58 return res.json(result)
61} 59}
62 60
63async function formatJobWithUnknownState (job: any) { 61async function formatJob (job: any, state?: JobState): Promise<Job> {
64 return formatJob(job, await job.getState())
65}
66
67function formatJob (job: any, state: JobState): Job {
68 const error = isArray(job.stacktrace) && job.stacktrace.length !== 0 62 const error = isArray(job.stacktrace) && job.stacktrace.length !== 0
69 ? job.stacktrace[0] 63 ? job.stacktrace[0]
70 : null 64 : null
71 65
72 return { 66 return {
73 id: job.id, 67 id: job.id,
74 state: state, 68 state: state || await job.getState(),
75 type: job.queue.name as JobType, 69 type: job.queue.name as JobType,
76 data: job.data, 70 data: job.data,
71 progress: await job.progress(),
77 error, 72 error,
78 createdAt: new Date(job.timestamp), 73 createdAt: new Date(job.timestamp),
79 finishedOn: new Date(job.finishedOn), 74 finishedOn: new Date(job.finishedOn),
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 6f7c186d9..a4d02908d 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,3 +1,4 @@
1import { Job } from 'bull'
1import * as ffmpeg from 'fluent-ffmpeg' 2import * as ffmpeg from 'fluent-ffmpeg'
2import { readFile, remove, writeFile } from 'fs-extra' 3import { readFile, remove, writeFile } from 'fs-extra'
3import { dirname, join } from 'path' 4import { dirname, join } from 'path'
@@ -124,6 +125,8 @@ interface BaseTranscodeOptions {
124 resolution: VideoResolution 125 resolution: VideoResolution
125 126
126 isPortraitMode?: boolean 127 isPortraitMode?: boolean
128
129 job?: Job
127} 130}
128 131
129interface HLSTranscodeOptions extends BaseTranscodeOptions { 132interface HLSTranscodeOptions extends BaseTranscodeOptions {
@@ -188,7 +191,7 @@ async function transcode (options: TranscodeOptions) {
188 191
189 command = await builders[options.type](command, options) 192 command = await builders[options.type](command, options)
190 193
191 await runCommand(command) 194 await runCommand(command, options.job)
192 195
193 await fixHLSPlaylistIfNeeded(options) 196 await fixHLSPlaylistIfNeeded(options)
194} 197}
@@ -611,11 +614,9 @@ function getFFmpeg (input: string, type: 'live' | 'vod') {
611 return command 614 return command
612} 615}
613 616
614async function runCommand (command: ffmpeg.FfmpegCommand, onEnd?: Function) { 617async function runCommand (command: ffmpeg.FfmpegCommand, job?: Job) {
615 return new Promise<void>((res, rej) => { 618 return new Promise<void>((res, rej) => {
616 command.on('error', (err, stdout, stderr) => { 619 command.on('error', (err, stdout, stderr) => {
617 if (onEnd) onEnd()
618
619 logger.error('Error in transcoding job.', { stdout, stderr }) 620 logger.error('Error in transcoding job.', { stdout, stderr })
620 rej(err) 621 rej(err)
621 }) 622 })
@@ -623,11 +624,18 @@ async function runCommand (command: ffmpeg.FfmpegCommand, onEnd?: Function) {
623 command.on('end', (stdout, stderr) => { 624 command.on('end', (stdout, stderr) => {
624 logger.debug('FFmpeg command ended.', { stdout, stderr }) 625 logger.debug('FFmpeg command ended.', { stdout, stderr })
625 626
626 if (onEnd) onEnd()
627
628 res() 627 res()
629 }) 628 })
630 629
630 if (job) {
631 command.on('progress', progress => {
632 if (!progress.percent) return
633
634 job.progress(Math.round(progress.percent))
635 .catch(err => logger.warn('Cannot set ffmpeg job progress.', { err }))
636 })
637 }
638
631 command.run() 639 command.run()
632 }) 640 })
633} 641}
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 20f8c3f50..083cec11a 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -44,20 +44,21 @@ async function processVideoTranscoding (job: Bull.Job) {
44 videoInputPath, 44 videoInputPath,
45 resolution: payload.resolution, 45 resolution: payload.resolution,
46 copyCodecs: payload.copyCodecs, 46 copyCodecs: payload.copyCodecs,
47 isPortraitMode: payload.isPortraitMode || false 47 isPortraitMode: payload.isPortraitMode || false,
48 job
48 }) 49 })
49 50
50 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) 51 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
51 } else if (payload.type === 'new-resolution') { 52 } else if (payload.type === 'new-resolution') {
52 await transcodeNewResolution(video, payload.resolution, payload.isPortraitMode || false) 53 await transcodeNewResolution(video, payload.resolution, payload.isPortraitMode || false, job)
53 54
54 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) 55 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload)
55 } else if (payload.type === 'merge-audio') { 56 } else if (payload.type === 'merge-audio') {
56 await mergeAudioVideofile(video, payload.resolution) 57 await mergeAudioVideofile(video, payload.resolution, job)
57 58
58 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) 59 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload)
59 } else { 60 } else {
60 const transcodeType = await optimizeOriginalVideofile(video) 61 const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job)
61 62
62 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload, transcodeType) 63 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload, transcodeType)
63 } 64 }
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index a6b79eaea..beef78b44 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,3 +1,4 @@
1import { Job } from 'bull'
1import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' 2import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
2import { basename, extname as extnameUtil, join } from 'path' 3import { basename, extname as extnameUtil, join } from 'path'
3import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 4import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
@@ -23,11 +24,10 @@ import { availableEncoders } from './video-transcoding-profiles'
23 */ 24 */
24 25
25// Optimize the original video file and replace it. The resolution is not changed. 26// Optimize the original video file and replace it. The resolution is not changed.
26async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFileArg?: MVideoFile) { 27async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: MVideoFile, job?: Job) {
27 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 28 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
28 const newExtname = '.mp4' 29 const newExtname = '.mp4'
29 30
30 const inputVideoFile = inputVideoFileArg || video.getMaxQualityFile()
31 const videoInputPath = getVideoFilePath(video, inputVideoFile) 31 const videoInputPath = getVideoFilePath(video, inputVideoFile)
32 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 32 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
33 33
@@ -44,7 +44,9 @@ async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFileA
44 availableEncoders, 44 availableEncoders,
45 profile: 'default', 45 profile: 'default',
46 46
47 resolution: inputVideoFile.resolution 47 resolution: inputVideoFile.resolution,
48
49 job
48 } 50 }
49 51
50 // Could be very long! 52 // Could be very long!
@@ -70,7 +72,7 @@ async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFileA
70} 72}
71 73
72// Transcode the original video file to a lower resolution. 74// Transcode the original video file to a lower resolution.
73async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean) { 75async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean, job: Job) {
74 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 76 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
75 const extname = '.mp4' 77 const extname = '.mp4'
76 78
@@ -96,7 +98,9 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR
96 availableEncoders, 98 availableEncoders,
97 profile: 'default', 99 profile: 'default',
98 100
99 resolution 101 resolution,
102
103 job
100 } 104 }
101 : { 105 : {
102 type: 'video' as 'video', 106 type: 'video' as 'video',
@@ -107,7 +111,9 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR
107 profile: 'default', 111 profile: 'default',
108 112
109 resolution, 113 resolution,
110 isPortraitMode: isPortrait 114 isPortraitMode: isPortrait,
115
116 job
111 } 117 }
112 118
113 await transcode(transcodeOptions) 119 await transcode(transcodeOptions)
@@ -116,7 +122,7 @@ async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoR
116} 122}
117 123
118// Merge an image with an audio file to create a video 124// Merge an image with an audio file to create a video
119async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution) { 125async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution, job: Job) {
120 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 126 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
121 const newExtname = '.mp4' 127 const newExtname = '.mp4'
122 128
@@ -140,7 +146,9 @@ async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: Video
140 profile: 'default', 146 profile: 'default',
141 147
142 audioPath: audioInputPath, 148 audioPath: audioInputPath,
143 resolution 149 resolution,
150
151 job
144 } 152 }
145 153
146 try { 154 try {
@@ -190,6 +198,7 @@ function generateHlsPlaylist (options: {
190 resolution: VideoResolution 198 resolution: VideoResolution
191 copyCodecs: boolean 199 copyCodecs: boolean
192 isPortraitMode: boolean 200 isPortraitMode: boolean
201 job?: Job
193}) { 202}) {
194 return generateHlsPlaylistCommon({ 203 return generateHlsPlaylistCommon({
195 video: options.video, 204 video: options.video,
@@ -197,7 +206,8 @@ function generateHlsPlaylist (options: {
197 copyCodecs: options.copyCodecs, 206 copyCodecs: options.copyCodecs,
198 isPortraitMode: options.isPortraitMode, 207 isPortraitMode: options.isPortraitMode,
199 inputPath: options.videoInputPath, 208 inputPath: options.videoInputPath,
200 type: 'hls' as 'hls' 209 type: 'hls' as 'hls',
210 job: options.job
201 }) 211 })
202} 212}
203 213
@@ -251,8 +261,10 @@ async function generateHlsPlaylistCommon (options: {
251 copyCodecs?: boolean 261 copyCodecs?: boolean
252 isAAC?: boolean 262 isAAC?: boolean
253 isPortraitMode: boolean 263 isPortraitMode: boolean
264
265 job?: Job
254}) { 266}) {
255 const { type, video, inputPath, resolution, copyCodecs, isPortraitMode, isAAC } = options 267 const { type, video, inputPath, resolution, copyCodecs, isPortraitMode, isAAC, job } = options
256 268
257 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 269 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
258 await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)) 270 await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid))
@@ -277,7 +289,9 @@ async function generateHlsPlaylistCommon (options: {
277 289
278 hlsPlaylist: { 290 hlsPlaylist: {
279 videoFilename 291 videoFilename
280 } 292 },
293
294 job
281 } 295 }
282 296
283 await transcode(transcodeOptions) 297 await transcode(transcodeOptions)
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index f9a6250c9..11d90c32f 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -23,6 +23,7 @@ export interface Job {
23 state: JobState 23 state: JobState
24 type: JobType 24 type: JobType
25 data: any 25 data: any
26 progress: number
26 error: any 27 error: any
27 createdAt: Date | string 28 createdAt: Date | string
28 finishedOn: Date | string 29 finishedOn: Date | string