From 764b1a14fc494f2cfd7ea590d2f07b01df65c7ad Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 23 Jul 2021 11:20:00 +0200 Subject: Use random names for VOD HLS playlists --- .../activitypub/videos/shared/abstract-builder.ts | 12 ++-- .../videos/shared/object-to-model-attributes.ts | 13 ++-- server/lib/hls.ts | 35 +++++------ server/lib/job-queue/handlers/video-file-import.ts | 3 +- server/lib/job-queue/handlers/video-live-ending.ts | 15 +++-- server/lib/job-queue/handlers/video-transcoding.ts | 3 +- server/lib/live/live-manager.ts | 26 ++++---- server/lib/live/shared/muxing-session.ts | 7 ++- .../lib/schedulers/videos-redundancy-scheduler.ts | 7 ++- server/lib/transcoding/video-transcoding.ts | 71 +++++++++++++--------- server/lib/video-paths.ts | 29 +++++++-- server/lib/video.ts | 4 +- 12 files changed, 131 insertions(+), 94 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index e89c94bcd..f995fe637 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -1,6 +1,6 @@ import { Transaction } from 'sequelize/types' import { checkUrlsSameHost } from '@server/helpers/activitypub' -import { deleteNonExistingModels } from '@server/helpers/database-utils' +import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' import { logger, LoggerTagsFn } from '@server/helpers/logger' import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' import { setVideoTags } from '@server/lib/video' @@ -111,8 +111,7 @@ export abstract class APVideoAbstractBuilder { const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) // Remove video files that do not exist anymore - const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t) - await Promise.all(destroyTasks) + await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t) // Update or add other one const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) @@ -124,13 +123,11 @@ export abstract class APVideoAbstractBuilder { const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) // Remove video playlists that do not exist anymore - const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t) - await Promise.all(destroyTasks) + await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) video.VideoStreamingPlaylists = [] for (const playlistAttributes of streamingPlaylistAttributes) { - const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) streamingPlaylistModel.Video = video @@ -163,8 +160,7 @@ export abstract class APVideoAbstractBuilder { const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) - const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t) - await Promise.all(destroyTasks) + await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t) // Update or add other one const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index 85548428c..1fa16295d 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -7,10 +7,11 @@ import { logger } from '@server/helpers/logger' import { getExtFromMimetype } from '@server/helpers/video' import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' import { generateTorrentFileName } from '@server/lib/video-paths' +import { VideoCaptionModel } from '@server/models/video/video-caption' import { VideoFileModel } from '@server/models/video/video-file' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' import { FilteredModelAttributes } from '@server/types' -import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models' +import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models' import { ActivityHashTagObject, ActivityMagnetUrlObject, @@ -23,7 +24,6 @@ import { VideoPrivacy, VideoStreamingPlaylistType } from '@shared/models' -import { VideoCaptionModel } from '@server/models/video/video-caption' function getThumbnailFromIcons (videoObject: VideoObject) { let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) @@ -80,8 +80,8 @@ function getFileAttributesFromUrl ( const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) const resolution = fileUrl.height - const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id - const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null + const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id + const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null const attribute = { extname, @@ -130,8 +130,13 @@ function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: const attribute = { type: VideoStreamingPlaylistType.HLS, + + playlistFilename: basename(playlistUrlObject.href), playlistUrl: playlistUrlObject.href, + + segmentsSha256Filename: basename(segmentsSha256UrlObject.href), segmentsSha256Url: segmentsSha256UrlObject.href, + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, videoId: video.id, diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 212bd095b..32b02bc26 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,7 +1,7 @@ import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' import { flatten, uniq } from 'lodash' import { basename, dirname, join } from 'path' -import { MVideoWithFile } from '@server/types/models' +import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' import { sha256 } from '../helpers/core-utils' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' import { logger } from '../helpers/logger' @@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { getVideoFilePath } from './video-paths' +import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -22,27 +22,29 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { await sequelizeTypescript.transaction(async t => { const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) - playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) + playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + await playlist.save({ transaction: t }) }) } } -async function updateMasterHLSPlaylist (video: MVideoWithFile) { +async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) - const streamingPlaylist = video.getHLSPlaylist() - for (const file of streamingPlaylist.VideoFiles) { - const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution) + const masterPlaylistPath = join(directory, playlist.playlistFilename) + + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) // If we did not generated a playlist for this resolution, skip const filePlaylistPath = join(directory, playlistFilename) if (await pathExists(filePlaylistPath) === false) continue - const videoFilePath = getVideoFilePath(streamingPlaylist, file) + const videoFilePath = getVideoFilePath(playlist, file) const size = await getVideoStreamSize(videoFilePath) @@ -66,23 +68,22 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) { await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') } -async function updateSha256VODSegments (video: MVideoWithFile) { +async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { const json: { [filename: string]: { [range: string]: string } } = {} const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) - const hlsPlaylist = video.getHLSPlaylist() // For all the resolutions available for this video - for (const file of hlsPlaylist.VideoFiles) { + for (const file of playlist.VideoFiles) { const rangeHashes: { [range: string]: string } = {} - const videoPath = getVideoFilePath(hlsPlaylist, file) - const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + const videoPath = getVideoFilePath(playlist, file) + const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename)) // Maybe the playlist is not generated for this resolution yet - if (!await pathExists(playlistPath)) continue + if (!await pathExists(resolutionPlaylistPath)) continue - const playlistContent = await readFile(playlistPath) + const playlistContent = await readFile(resolutionPlaylistPath) const ranges = getRangesFromPlaylist(playlistContent.toString()) const fd = await open(videoPath, 'r') @@ -98,7 +99,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) { json[videoFilename] = rangeHashes } - const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) await outputJSON(outputPath, json) } diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 1783f206a..4d199f247 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -61,8 +61,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { if (currentVideoFile) { // Remove old file and old torrent - await video.removeFile(currentVideoFile) - await currentVideoFile.removeTorrent() + await video.removeFileAndTorrent(currentVideoFile) // Remove the old video file from the array video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 9eba41bf8..386ccdc7b 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -7,12 +7,12 @@ import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server import { generateVideoMiniature } from '@server/lib/thumbnail' import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' import { publishAndFederateIfNeeded } from '@server/lib/video' -import { getHLSDirectory } from '@server/lib/video-paths' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MVideo, MVideoLive } from '@server/types/models' +import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger } from '../../../helpers/logger' @@ -43,7 +43,7 @@ async function processVideoLiveEnding (job: Bull.Job) { return cleanupLive(video, streamingPlaylist) } - return saveLive(video, live) + return saveLive(video, live, streamingPlaylist) } // --------------------------------------------------------------------------- @@ -54,14 +54,14 @@ export { // --------------------------------------------------------------------------- -async function saveLive (video: MVideo, live: MVideoLive) { +async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { const hlsDirectory = getHLSDirectory(video, false) const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY) const rootFiles = await readdir(hlsDirectory) const playlistFiles = rootFiles.filter(file => { - return file.endsWith('.m3u8') && file !== 'master.m3u8' + return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename }) await cleanupLiveFiles(hlsDirectory) @@ -80,7 +80,12 @@ async function saveLive (video: MVideo, live: MVideoLive) { const hlsPlaylist = videoWithFiles.getHLSPlaylist() await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) + + // Reset playlist hlsPlaylist.VideoFiles = [] + hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename() + hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() + await hlsPlaylist.save() let durationDone = false diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index f5ba6f435..36d9594af 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -125,8 +125,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { // Remove webtorrent files if not enabled for (const file of video.VideoFiles) { - await video.removeFile(file) - await file.removeTorrent() + await video.removeFileAndTorrent(file) await file.destroy() } diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index da764e009..f106d69fb 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -4,16 +4,17 @@ import { isTestInstance } from '@server/helpers/core-utils' import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' +import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME } from '@server/initializers/constants' import { UserModel } from '@server/models/user/user' import { VideoModel } from '@server/models/video/video' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' +import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' import { VideoState, VideoStreamingPlaylistType } from '@shared/models' import { federateVideoIfNeeded } from '../activitypub/videos' import { JobQueue } from '../job-queue' import { PeerTubeSocket } from '../peertube-socket' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' import { LiveQuotaStore } from './live-quota-store' import { LiveSegmentShaStore } from './live-segment-sha-store' import { cleanupLive } from './live-utils' @@ -392,19 +393,18 @@ class LiveManager { return resolutionsEnabled.concat([ originResolution ]) } - private async createLivePlaylist (video: MVideo, allResolutions: number[]) { - const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) - const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ - videoId: video.id, - playlistUrl, - segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), - p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions), - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, + private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise { + const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video) - type: VideoStreamingPlaylistType.HLS - }, { returning: true }) as [ MStreamingPlaylist, boolean ] + playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) - return Object.assign(videoStreamingPlaylist, { Video: video }) + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + playlist.type = VideoStreamingPlaylistType.HLS + + playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) + + return playlist.save() } static get Instance () { diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index 26467f060..709d6c615 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts @@ -112,13 +112,16 @@ class MuxingSession extends EventEmitter { this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED ? await getLiveTranscodingCommand({ rtmpUrl: this.rtmpUrl, + outPath, + masterPlaylistName: this.streamingPlaylist.playlistFilename, + resolutions: this.allResolutions, fps: this.fps, availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), profile: CONFIG.LIVE.TRANSCODING.PROFILE }) - : getLiveMuxingCommand(this.rtmpUrl, outPath) + : getLiveMuxingCommand(this.rtmpUrl, outPath, this.streamingPlaylist.playlistFilename) logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) @@ -182,7 +185,7 @@ class MuxingSession extends EventEmitter { } private watchMasterFile (outPath: string) { - this.masterWatcher = chokidar.watch(outPath + '/master.m3u8') + this.masterWatcher = chokidar.watch(outPath + '/' + this.streamingPlaylist.playlistFilename) this.masterWatcher.on('add', async () => { this.emit('master-playlist-created', { videoId: this.videoId }) diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index b5a5eb697..103ab1fab 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -267,7 +267,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) - await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) + const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) + await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ expiresOn, @@ -282,7 +283,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await sendCreateCacheFile(serverActor, video, createdModel) - logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) + logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url) } private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { @@ -330,7 +331,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` - return `${object.VideoStreamingPlaylist.playlistUrl}` + return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` } private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index d70f7f474..d2a556360 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts @@ -10,11 +10,18 @@ import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' import { logger } from '../../helpers/logger' import { CONFIG } from '../../initializers/config' -import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants' +import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' import { VideoFileModel } from '../../models/video/video-file' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' -import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths' +import { + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + generateHLSVideoFilename, + generateWebTorrentVideoFilename, + getHlsResolutionPlaylistFilename, + getVideoFilePath +} from '../video-paths' import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' /** @@ -272,14 +279,14 @@ async function generateHlsPlaylistCommon (options: { await ensureDir(videoTranscodedBasePath) const videoFilename = generateHLSVideoFilename(resolution) - const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution) - const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename) + const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) + const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename) const transcodeOptions = { type, inputPath, - outputPath: playlistFileTranscodePath, + outputPath: resolutionPlaylistFileTranscodePath, availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), profile: CONFIG.TRANSCODING.PROFILE, @@ -299,19 +306,23 @@ async function generateHlsPlaylistCommon (options: { await transcode(transcodeOptions) - const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) - // Create or update the playlist - const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ - videoId: video.id, - playlistUrl, - segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), - p2pMediaLoaderInfohashes: [], - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, + const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video) + + if (!playlist.playlistFilename) { + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + } + + if (!playlist.segmentsSha256Filename) { + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) + } + + playlist.p2pMediaLoaderInfohashes = [] + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - type: VideoStreamingPlaylistType.HLS - }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ] - videoStreamingPlaylist.Video = video + playlist.type = VideoStreamingPlaylistType.HLS + + await playlist.save() // Build the new playlist file const extname = extnameUtil(videoFilename) @@ -321,18 +332,18 @@ async function generateHlsPlaylistCommon (options: { size: 0, filename: videoFilename, fps: -1, - videoStreamingPlaylistId: videoStreamingPlaylist.id + videoStreamingPlaylistId: playlist.id }) - const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile) + const videoFilePath = getVideoFilePath(playlist, newVideoFile) // Move files from tmp transcoded directory to the appropriate place const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) await ensureDir(baseHlsDirectory) // Move playlist file - const playlistPath = join(baseHlsDirectory, playlistFilename) - await move(playlistFileTranscodePath, playlistPath, { overwrite: true }) + const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) + await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) // Move video file await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) @@ -342,20 +353,20 @@ async function generateHlsPlaylistCommon (options: { newVideoFile.fps = await getVideoFileFPS(videoFilePath) newVideoFile.metadata = await getMetadataFromFile(videoFilePath) - await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) + await createTorrentAndSetInfoHash(playlist, newVideoFile) await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) - videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles') - videoStreamingPlaylist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes( - playlistUrl, videoStreamingPlaylist.VideoFiles - ) - await videoStreamingPlaylist.save() + const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo + playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') + playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + + await playlist.save() - video.setHLSPlaylist(videoStreamingPlaylist) + video.setHLSPlaylist(playlist) - await updateMasterHLSPlaylist(video) - await updateSha256VODSegments(video) + await updateMasterHLSPlaylist(video, playlistWithFiles) + await updateSha256VODSegments(video, playlistWithFiles) - return playlistPath + return resolutionPlaylistPath } diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index b7068190c..1e4382108 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts @@ -4,19 +4,16 @@ import { CONFIG } from '@server/initializers/config' import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' import { buildUUID } from '@server/helpers/uuid' +import { removeFragmentedMP4Ext } from '@shared/core-utils' // ################## Video file name ################## function generateWebTorrentVideoFilename (resolution: number, extname: string) { - const uuid = buildUUID() - - return uuid + '-' + resolution + extname + return buildUUID() + '-' + resolution + extname } function generateHLSVideoFilename (resolution: number) { - const uuid = buildUUID() - - return `${uuid}-${resolution}-fragmented.mp4` + return `${buildUUID()}-${resolution}-fragmented.mp4` } function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { @@ -54,6 +51,23 @@ function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { return join(baseDir, video.uuid) } +function getHlsResolutionPlaylistFilename (videoFilename: string) { + // Video file name already contain resolution + return removeFragmentedMP4Ext(videoFilename) + '.m3u8' +} + +function generateHLSMasterPlaylistFilename (isLive = false) { + if (isLive) return 'master.m3u8' + + return buildUUID() + '-master.m3u8' +} + +function generateHlsSha256SegmentsFilename (isLive = false) { + if (isLive) return 'segments-sha256.json' + + return buildUUID() + '-segments-sha256.json' +} + // ################## Torrents ################## function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { @@ -91,6 +105,9 @@ export { getTorrentFilePath, getHLSDirectory, + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + getHlsResolutionPlaylistFilename, getLocalVideoFileMetadataUrl, diff --git a/server/lib/video.ts b/server/lib/video.ts index daf998704..61fee4949 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database' import { TagModel } from '@server/models/video/tag' import { VideoModel } from '@server/models/video/video' import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' +import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' import { federateVideoIfNeeded } from './activitypub/videos' import { JobQueue } from './job-queue/job-queue' @@ -105,7 +105,7 @@ async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) { } } -async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) { +async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { let dataInput: VideoTranscodingPayload if (videoFile.isAudio()) { -- cgit v1.2.3