From: Chocobozzz Date: Tue, 27 Oct 2020 15:06:24 +0000 (+0100) Subject: Fix replay saving X-Git-Tag: v3.0.0-rc.1~420 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=31c82cd914e13dbf53280d0aad0740d70c414441;hp=da0310f821b039fea1fcbf8aea49e1d2279ba98e;p=github%2FChocobozzz%2FPeerTube.git Fix replay saving --- diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 2f167a580..b063cedcb 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -8,6 +8,7 @@ import { CONFIG } from '../initializers/config' import { FFMPEG_NICE, VIDEO_LIVE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' import { processImage } from './image-utils' import { logger } from './logger' +import { concat } from 'lodash' /** * A toolbox to play with audio @@ -424,17 +425,40 @@ function runLiveMuxing (rtmpUrl: string, outPath: string, deleteSegments: boolea return command } -function hlsPlaylistToFragmentedMP4 (playlistPath: string, outputPath: string) { - const command = getFFmpeg(playlistPath) +async function hlsPlaylistToFragmentedMP4 (hlsDirectory: string, segmentFiles: string[], outputPath: string) { + const concatFile = 'concat.txt' + const concatFilePath = join(hlsDirectory, concatFile) + const content = segmentFiles.map(f => 'file ' + f) + .join('\n') + + await writeFile(concatFilePath, content + '\n') + + const command = getFFmpeg(concatFilePath) + command.inputOption('-safe 0') + command.inputOption('-f concat') command.outputOption('-c copy') command.output(outputPath) command.run() + function cleaner () { + remove(concatFile) + .catch(err => logger.error('Cannot remove concat file in %s.', hlsDirectory, { err })) + } + return new Promise((res, rej) => { - command.on('error', err => rej(err)) - command.on('end', () => res()) + command.on('error', err => { + cleaner() + + rej(err) + }) + + command.on('end', () => { + cleaner() + + res() + }) }) } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 065012b32..f0d614112 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -154,7 +154,7 @@ const JOB_CONCURRENCY: { [id in JobType]: number } = { 'videos-views': 1, 'activitypub-refresher': 1, 'video-redundancy': 1, - 'video-live-ending': 1 + 'video-live-ending': 10 } const JOB_TTL: { [id in JobType]: number } = { 'activitypub-http-broadcast': 60000 * 10, // 10 minutes @@ -736,6 +736,8 @@ if (isTestInstance() === true) { OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2 PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000 + + VIDEO_LIVE.CLEANUP_DELAY = 10000 } updateWebserverUrls() diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 1a9a36129..cd5bb1d1c 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -7,7 +7,7 @@ import { generateHlsPlaylist } from '@server/lib/video-transcoding' import { VideoModel } from '@server/models/video/video' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylist, MVideo } from '@server/types/models' +import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' import { VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger } from '../../../helpers/logger' @@ -27,7 +27,7 @@ async function processVideoLiveEnding (job: Bull.Job) { return cleanupLive(video, streamingPlaylist) } - return saveLive(video, streamingPlaylist) + return saveLive(video, live) } // --------------------------------------------------------------------------- @@ -38,33 +38,47 @@ export { // --------------------------------------------------------------------------- -async function saveLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - const videoFiles = await streamingPlaylist.get('VideoFiles') +async function saveLive (video: MVideo, live: MVideoLive) { const hlsDirectory = getHLSDirectory(video, false) + const files = await readdir(hlsDirectory) + + const playlistFiles = files.filter(f => f.endsWith('.m3u8') && f !== 'master.m3u8') + const resolutions: number[] = [] + + for (const playlistFile of playlistFiles) { + const playlistPath = join(hlsDirectory, playlistFile) + const { videoFileResolution } = await getVideoFileResolution(playlistPath) - for (const videoFile of videoFiles) { - const playlistPath = join(hlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(videoFile.resolution)) + const mp4TmpName = buildMP4TmpName(videoFileResolution) - const mp4TmpName = buildMP4TmpName(videoFile.resolution) - await hlsPlaylistToFragmentedMP4(playlistPath, mp4TmpName) + // Playlist name is for example 3.m3u8 + // Segments names are 3-0.ts 3-1.ts etc + const shouldStartWith = playlistFile.replace(/\.m3u8$/, '') + '-' + + const segmentFiles = files.filter(f => f.startsWith(shouldStartWith) && f.endsWith('.ts')) + await hlsPlaylistToFragmentedMP4(hlsDirectory, segmentFiles, mp4TmpName) + + resolutions.push(videoFileResolution) } await cleanupLiveFiles(hlsDirectory) + await live.destroy() + video.isLive = false video.state = VideoState.TO_TRANSCODE await video.save() const videoWithFiles = await VideoModel.loadWithFiles(video.id) - for (const videoFile of videoFiles) { - const videoInputPath = buildMP4TmpName(videoFile.resolution) + for (const resolution of resolutions) { + const videoInputPath = buildMP4TmpName(resolution) const { isPortraitMode } = await getVideoFileResolution(videoInputPath) await generateHlsPlaylist({ video: videoWithFiles, videoInputPath, - resolution: videoFile.resolution, + resolution: resolution, copyCodecs: true, isPortraitMode }) @@ -103,5 +117,5 @@ async function cleanupLiveFiles (hlsDirectory: string) { } function buildMP4TmpName (resolution: number) { - return resolution + 'tmp.mp4' + return resolution + '-tmp.mp4' } diff --git a/server/models/account/user.ts b/server/models/account/user.ts index e850d1e6d..f64568c54 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -710,7 +710,7 @@ export class UserModel extends Model { required: true, include: [ { - attributes: [ 'id', 'videoId' ], + attributes: [], model: VideoLiveModel.unscoped(), required: true, where: { @@ -726,7 +726,7 @@ export class UserModel extends Model { ] } - return UserModel.findOne(query) + return UserModel.unscoped().findOne(query) } static generateUserQuotaBaseSQL (options: { diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 8493ab802..78fec5585 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -128,6 +128,7 @@ import { VideoStreamingPlaylistModel } from './video-streaming-playlist' import { VideoTagModel } from './video-tag' import { VideoViewModel } from './video-view' import { LiveManager } from '@server/lib/live-manager' +import { VideoLiveModel } from './video-live' export enum ScopeNames { AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS', @@ -725,6 +726,15 @@ export class VideoModel extends Model { }) VideoBlacklist: VideoBlacklistModel + @HasOne(() => VideoLiveModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoLive: VideoLiveModel + @HasOne(() => VideoImportModel, { foreignKey: { name: 'videoId',