aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-11-30 15:59:22 +0100
committerChocobozzz <me@florianbigard.com>2020-11-30 15:59:22 +0100
commit937581b8f61fe82fdac044e11740b9a4cb6d96b0 (patch)
tree9b9274e2e4696cf167a86691dfc35c024cd254ba /server/lib
parentd605328a300dca095fa93d7f03fa104573e5a3fa (diff)
downloadPeerTube-937581b8f61fe82fdac044e11740b9a4cb6d96b0.tar.gz
PeerTube-937581b8f61fe82fdac044e11740b9a4cb6d96b0.tar.zst
PeerTube-937581b8f61fe82fdac044e11740b9a4cb6d96b0.zip
Fix high CPU with long live when save replay is true
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/job-queue/handlers/video-live-ending.ts32
-rw-r--r--server/lib/live-manager.ts28
2 files changed, 45 insertions, 15 deletions
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts
index 447744224..0d2bcaa28 100644
--- a/server/lib/job-queue/handlers/video-live-ending.ts
+++ b/server/lib/job-queue/handlers/video-live-ending.ts
@@ -1,5 +1,5 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { readdir, remove } from 'fs-extra' 2import { move, readdir, remove } from 'fs-extra'
3import { join } from 'path' 3import { join } from 'path'
4import { hlsPlaylistToFragmentedMP4 } from '@server/helpers/ffmpeg-utils' 4import { hlsPlaylistToFragmentedMP4 } from '@server/helpers/ffmpeg-utils'
5import { getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 5import { getDurationFromVideoFile, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
@@ -14,6 +14,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
14import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' 14import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models'
15import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' 15import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
16import { logger } from '../../../helpers/logger' 16import { logger } from '../../../helpers/logger'
17import { VIDEO_LIVE } from '@server/initializers/constants'
17 18
18async function processVideoLiveEnding (job: Bull.Job) { 19async function processVideoLiveEnding (job: Bull.Job) {
19 const payload = job.data as VideoLiveEndingPayload 20 const payload = job.data as VideoLiveEndingPayload
@@ -53,24 +54,40 @@ export {
53 54
54async function saveLive (video: MVideo, live: MVideoLive) { 55async function saveLive (video: MVideo, live: MVideoLive) {
55 const hlsDirectory = getHLSDirectory(video, false) 56 const hlsDirectory = getHLSDirectory(video, false)
56 const files = await readdir(hlsDirectory) 57 const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
58
59 const rootFiles = await readdir(hlsDirectory)
60
61 const playlistFiles: string[] = []
62
63 for (const file of rootFiles) {
64 if (file.endsWith('.m3u8') !== true) continue
65
66 await move(join(hlsDirectory, file), join(replayDirectory, file))
67
68 if (file !== 'master.m3u8') {
69 playlistFiles.push(file)
70 }
71 }
72
73 const replayFiles = await readdir(replayDirectory)
57 74
58 const playlistFiles = files.filter(f => f.endsWith('.m3u8') && f !== 'master.m3u8')
59 const resolutions: number[] = [] 75 const resolutions: number[] = []
60 let duration: number 76 let duration: number
61 77
62 for (const playlistFile of playlistFiles) { 78 for (const playlistFile of playlistFiles) {
63 const playlistPath = join(hlsDirectory, playlistFile) 79 const playlistPath = join(replayDirectory, playlistFile)
64 const { videoFileResolution } = await getVideoFileResolution(playlistPath) 80 const { videoFileResolution } = await getVideoFileResolution(playlistPath)
65 81
82 // Put the final mp4 in the hls directory, and not in the replay directory
66 const mp4TmpPath = buildMP4TmpPath(hlsDirectory, videoFileResolution) 83 const mp4TmpPath = buildMP4TmpPath(hlsDirectory, videoFileResolution)
67 84
68 // Playlist name is for example 3.m3u8 85 // Playlist name is for example 3.m3u8
69 // Segments names are 3-0.ts 3-1.ts etc 86 // Segments names are 3-0.ts 3-1.ts etc
70 const shouldStartWith = playlistFile.replace(/\.m3u8$/, '') + '-' 87 const shouldStartWith = playlistFile.replace(/\.m3u8$/, '') + '-'
71 88
72 const segmentFiles = files.filter(f => f.startsWith(shouldStartWith) && f.endsWith('.ts')) 89 const segmentFiles = replayFiles.filter(f => f.startsWith(shouldStartWith) && f.endsWith('.ts'))
73 await hlsPlaylistToFragmentedMP4(hlsDirectory, segmentFiles, mp4TmpPath) 90 await hlsPlaylistToFragmentedMP4(replayDirectory, segmentFiles, mp4TmpPath)
74 91
75 if (!duration) { 92 if (!duration) {
76 duration = await getDurationFromVideoFile(mp4TmpPath) 93 duration = await getDurationFromVideoFile(mp4TmpPath)
@@ -143,7 +160,8 @@ async function cleanupLiveFiles (hlsDirectory: string) {
143 filename.endsWith('.m3u8') || 160 filename.endsWith('.m3u8') ||
144 filename.endsWith('.mpd') || 161 filename.endsWith('.mpd') ||
145 filename.endsWith('.m4s') || 162 filename.endsWith('.m4s') ||
146 filename.endsWith('.tmp') 163 filename.endsWith('.tmp') ||
164 filename === VIDEO_LIVE.REPLAY_DIRECTORY
147 ) { 165 ) {
148 const p = join(hlsDirectory, filename) 166 const p = join(hlsDirectory, filename)
149 167
diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts
index d63e79dfc..d201465fa 100644
--- a/server/lib/live-manager.ts
+++ b/server/lib/live-manager.ts
@@ -1,8 +1,8 @@
1 1
2import * as chokidar from 'chokidar' 2import * as chokidar from 'chokidar'
3import { FfmpegCommand } from 'fluent-ffmpeg' 3import { FfmpegCommand } from 'fluent-ffmpeg'
4import { ensureDir, stat } from 'fs-extra' 4import { copy, ensureDir, stat } from 'fs-extra'
5import { basename } from 'path' 5import { basename, join } from 'path'
6import { isTestInstance } from '@server/helpers/core-utils' 6import { isTestInstance } from '@server/helpers/core-utils'
7import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils' 7import { getLiveMuxingCommand, getLiveTranscodingCommand } from '@server/helpers/ffmpeg-utils'
8import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' 8import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
@@ -25,6 +25,7 @@ import { getHLSDirectory } from './video-paths'
25import { availableEncoders } from './video-transcoding-profiles' 25import { availableEncoders } from './video-transcoding-profiles'
26 26
27import memoizee = require('memoizee') 27import memoizee = require('memoizee')
28import { mkdir } from 'fs'
28const NodeRtmpServer = require('node-media-server/node_rtmp_server') 29const NodeRtmpServer = require('node-media-server/node_rtmp_server')
29const context = require('node-media-server/node_core_ctx') 30const context = require('node-media-server/node_core_ctx')
30const nodeMediaServerLogger = require('node-media-server/node_core_logger') 31const nodeMediaServerLogger = require('node-media-server/node_core_logger')
@@ -261,8 +262,13 @@ class LiveManager {
261 const outPath = getHLSDirectory(videoLive.Video) 262 const outPath = getHLSDirectory(videoLive.Video)
262 await ensureDir(outPath) 263 await ensureDir(outPath)
263 264
265 const replayDirectory = join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY)
266
267 if (videoLive.saveReplay === true) {
268 await ensureDir(replayDirectory)
269 }
270
264 const videoUUID = videoLive.Video.uuid 271 const videoUUID = videoLive.Video.uuid
265 const deleteSegments = videoLive.saveReplay === false
266 272
267 const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED 273 const ffmpegExec = CONFIG.LIVE.TRANSCODING.ENABLED
268 ? await getLiveTranscodingCommand({ 274 ? await getLiveTranscodingCommand({
@@ -270,11 +276,10 @@ class LiveManager {
270 outPath, 276 outPath,
271 resolutions: allResolutions, 277 resolutions: allResolutions,
272 fps, 278 fps,
273 deleteSegments,
274 availableEncoders, 279 availableEncoders,
275 profile: 'default' 280 profile: 'default'
276 }) 281 })
277 : getLiveMuxingCommand(rtmpUrl, outPath, deleteSegments) 282 : getLiveMuxingCommand(rtmpUrl, outPath)
278 283
279 logger.info('Running live muxing/transcoding for %s.', videoUUID) 284 logger.info('Running live muxing/transcoding for %s.', videoUUID)
280 this.transSessions.set(sessionId, ffmpegExec) 285 this.transSessions.set(sessionId, ffmpegExec)
@@ -284,11 +289,18 @@ class LiveManager {
284 const segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} 289 const segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
285 const playlistIdMatcher = /^([\d+])-/ 290 const playlistIdMatcher = /^([\d+])-/
286 291
287 const processHashSegments = (segmentsToProcess: string[]) => { 292 const processSegments = (segmentsToProcess: string[]) => {
288 // Add sha hash of previous segments, because ffmpeg should have finished generating them 293 // Add sha hash of previous segments, because ffmpeg should have finished generating them
289 for (const previousSegment of segmentsToProcess) { 294 for (const previousSegment of segmentsToProcess) {
290 this.addSegmentSha(videoUUID, previousSegment) 295 this.addSegmentSha(videoUUID, previousSegment)
291 .catch(err => logger.error('Cannot add sha segment of video %s -> %s.', videoUUID, previousSegment, { err })) 296 .catch(err => logger.error('Cannot add sha segment of video %s -> %s.', videoUUID, previousSegment, { err }))
297
298 if (videoLive.saveReplay) {
299 const segmentName = basename(previousSegment)
300
301 copy(previousSegment, join(outPath, VIDEO_LIVE.REPLAY_DIRECTORY, segmentName))
302 .catch(err => logger.error('Cannot copy segment %s to repay directory.', previousSegment, { err }))
303 }
292 } 304 }
293 } 305 }
294 306
@@ -298,7 +310,7 @@ class LiveManager {
298 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0] 310 const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
299 311
300 const segmentsToProcess = segmentsToProcessPerPlaylist[playlistId] || [] 312 const segmentsToProcess = segmentsToProcessPerPlaylist[playlistId] || []
301 processHashSegments(segmentsToProcess) 313 processSegments(segmentsToProcess)
302 314
303 segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] 315 segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
304 316
@@ -369,7 +381,7 @@ class LiveManager {
369 .then(() => { 381 .then(() => {
370 // Process remaining segments hash 382 // Process remaining segments hash
371 for (const key of Object.keys(segmentsToProcessPerPlaylist)) { 383 for (const key of Object.keys(segmentsToProcessPerPlaylist)) {
372 processHashSegments(segmentsToProcessPerPlaylist[key]) 384 processSegments(segmentsToProcessPerPlaylist[key])
373 } 385 }
374 }) 386 })
375 .catch(err => logger.error('Cannot close watchers of %s or process remaining hash segments.', outPath, { err })) 387 .catch(err => logger.error('Cannot close watchers of %s or process remaining hash segments.', outPath, { err }))