X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Fhls.ts;h=20754219f22c1b7f402bdbd86674d0e94d7abccd;hb=15eb9e5b8fe9a29201567ab41f730fd39c1a14b8;hp=d969549b86d3909df2eaba65a376f9d966fbf203;hpb=ad5db1044c8599eaaaa2a578b350777ae996b068;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/hls.ts b/server/lib/hls.ts index d969549b8..20754219f 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,18 +1,21 @@ import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' import { flatten, uniq } from 'lodash' +import PQueue from 'p-queue' import { basename, dirname, join } from 'path' -import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' -import { sha256 } from '../helpers/core-utils' -import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' +import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' +import { sha256 } from '@shared/extra-utils' +import { VideoStorage } from '@shared/models' +import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg' import { logger } from '../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' import { generateRandomString } from '../helpers/utils' import { CONFIG } from '../initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' +import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { getHlsResolutionPlaylistFilename } from './paths' +import { storeHLSFile } from './object-storage' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' import { VideoPathManager } from './video-path-manager' async function updateStreamingPlaylistsInfohashesIfNeeded () { @@ -31,71 +34,123 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { } } -async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { - const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] +async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { + let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) + playlistWithFiles = await updateSha256VODSegments(video, playlist) - for (const file of playlist.VideoFiles) { - const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) + // Refresh playlist, operations can take some time + playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) + playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + await playlistWithFiles.save() - await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { - const size = await getVideoStreamSize(videoFilePath) + video.setHLSPlaylist(playlistWithFiles) +} - const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) - const resolution = `RESOLUTION=${size.width}x${size.height}` +// --------------------------------------------------------------------------- - let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` - if (file.fps) line += ',FRAME-RATE=' + file.fps +// Avoid concurrency issues when updating streaming playlist files +const playlistFilesQueue = new PQueue({ concurrency: 1 }) - const codecs = await Promise.all([ - getVideoStreamCodec(videoFilePath), - getAudioStreamCodec(videoFilePath) - ]) +function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - masterPlaylists.push(line) - masterPlaylists.push(playlistFilename) - }) - } + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) + + await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { + const size = await getVideoStreamDimensionsInfo(videoFilePath) + + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` + + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps + + const codecs = await Promise.all([ + getVideoStreamCodec(videoFilePath), + getAudioStreamCodec(videoFilePath) + ]) - await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { - return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') + line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + + masterPlaylists.push(line) + masterPlaylists.push(playlistFilename) + }) + } + + if (playlist.playlistFilename) { + await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) + } + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + + const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) + await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') + + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename) + await remove(masterPlaylistPath) + } + + return playlist.save() }) } -async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { - const json: { [filename: string]: { [range: string]: string } } = {} +// --------------------------------------------------------------------------- + +async function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const json: { [filename: string]: { [range: string]: string } } = {} + + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - // For all the resolutions available for this video - for (const file of playlist.VideoFiles) { - const rangeHashes: { [range: string]: string } = {} - const fileWithPlaylist = file.withVideoOrPlaylist(playlist) + // For all the resolutions available for this video + for (const file of playlist.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { + await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { - const playlistContent = await readFile(resolutionPlaylistPath) - const ranges = getRangesFromPlaylist(playlistContent.toString()) + return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { + const playlistContent = await readFile(resolutionPlaylistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - const fd = await open(videoPath, 'r') - for (const range of ranges) { - const buf = Buffer.alloc(range.length) - await read(fd, buf, 0, range.length, range.offset) + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) - rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) - } - await close(fd) + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) - const videoFilename = file.filename - json[videoFilename] = rangeHashes + const videoFilename = file.filename + json[videoFilename] = rangeHashes + }) }) - }) - } + } - const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) - await outputJSON(outputPath, json) + if (playlist.segmentsSha256Filename) { + await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) + } + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) + + const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) + await outputJSON(outputPath, json) + + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename) + await remove(outputPath) + } + + return playlist.save() + }) } +// --------------------------------------------------------------------------- + async function buildSha256Segment (segmentPath: string) { const buf = await readFile(segmentPath) return sha256(buf) @@ -130,7 +185,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, for (const fileUrl of fileUrls) { const destPath = join(tmpDirectory, basename(fileUrl)) - await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit }) + await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) const { size } = await stat(destPath) remainingBodyKBLimit -= (size / 1000) @@ -179,7 +234,8 @@ export { updateSha256VODSegments, buildSha256Segment, downloadPlaylistSegments, - updateStreamingPlaylistsInfohashesIfNeeded + updateStreamingPlaylistsInfohashesIfNeeded, + updatePlaylistAfterFileChange } // ---------------------------------------------------------------------------