X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Fhls.ts;h=fc1d7e1b0cc613f3707083a0ae6706ed42cddf76;hb=e901579b00fbcd8fc0f7b45fd841636329148a34;hp=76380b1f23e9730a4603225d007dab7ae067b40e;hpb=d7b1c7b4f1a9099eac61f944249f2cdb4f94415e;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 76380b1f2..fc1d7e1b0 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,18 +1,24 @@ +import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' +import { flatten } from 'lodash' +import PQueue from 'p-queue' import { basename, dirname, join } from 'path' -import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' -import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' -import { getVideoStreamSize, getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg-utils' -import { sha256 } from '../helpers/core-utils' -import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' +import { uniqify, uuidRegex } from '@shared/core-utils' +import { sha256 } from '@shared/extra-utils' +import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' +import { VideoStorage } from '@shared/models' +import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' import { logger } from '../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' import { generateRandomString } from '../helpers/utils' -import { flatten, uniq } from 'lodash' -import { VideoFileModel } from '../models/video/video-file' import { CONFIG } from '../initializers/config' +import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' import { sequelizeTypescript } from '../initializers/database' -import { MVideoWithFile } from '@server/types/models' -import { getVideoFilename, getVideoFilePath } from './video-paths' +import { VideoFileModel } from '../models/video/video-file' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import { storeHLSFileFromFilename } from './object-storage' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' +import { VideoPathManager } from './video-path-manager' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -22,107 +28,147 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { await sequelizeTypescript.transaction(async t => { const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) - playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) + playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + await playlist.save({ transaction: t }) }) } } -async function updateMasterHLSPlaylist (video: MVideoWithFile) { - const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) - const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) - const streamingPlaylist = video.getHLSPlaylist() +async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { + try { + let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) + playlistWithFiles = await updateSha256VODSegments(video, playlist) - for (const file of streamingPlaylist.VideoFiles) { - // If we did not generated a playlist for this resolution, skip - const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) - if (await pathExists(filePlaylistPath) === false) continue + // Refresh playlist, operations can take some time + playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) + playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + await playlistWithFiles.save() - const videoFilePath = getVideoFilePath(streamingPlaylist, file) + video.setHLSPlaylist(playlistWithFiles) + } catch (err) { + logger.info('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) + } +} - const size = await getVideoStreamSize(videoFilePath) +// --------------------------------------------------------------------------- - const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) - const resolution = `RESOLUTION=${size.width}x${size.height}` +// Avoid concurrency issues when updating streaming playlist files +const playlistFilesQueue = new PQueue({ concurrency: 1 }) - let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` - if (file.fps) line += ',FRAME-RATE=' + file.fps +function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - const videoCodec = await getVideoStreamCodec(videoFilePath) - line += `,CODECS="${videoCodec}` + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - const audioCodec = await getAudioStreamCodec(videoFilePath) - if (audioCodec) line += `,${audioCodec}` + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - line += '"' + await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { + const size = await getVideoStreamDimensionsInfo(videoFilePath) - masterPlaylists.push(line) - masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) - } + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` + + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps + + const codecs = await Promise.all([ + getVideoStreamCodec(videoFilePath), + getAudioStreamCodec(videoFilePath) + ]) + + line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + + masterPlaylists.push(line) + masterPlaylists.push(playlistFilename) + }) + } + + if (playlist.playlistFilename) { + await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) + } + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + + const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) + await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') - await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) + await remove(masterPlaylistPath) + } + + return playlist.save() + }) } -async function updateSha256Segments (video: MVideoWithFile) { - const json: { [filename: string]: { [range: string]: string } } = {} +// --------------------------------------------------------------------------- + +function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const json: { [filename: string]: { [range: string]: string } } = {} - const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) - const hlsPlaylist = video.getHLSPlaylist() + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - // For all the resolutions available for this video - for (const file of hlsPlaylist.VideoFiles) { - const rangeHashes: { [range: string]: string } = {} + // For all the resolutions available for this video + for (const file of playlist.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - const videoPath = getVideoFilePath(hlsPlaylist, file) - const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - // Maybe the playlist is not generated for this resolution yet - if (!await pathExists(playlistPath)) continue + return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { + const playlistContent = await readFile(resolutionPlaylistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - const playlistContent = await readFile(playlistPath) - const ranges = getRangesFromPlaylist(playlistContent.toString()) + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) - const fd = await open(videoPath, 'r') - for (const range of ranges) { - const buf = Buffer.alloc(range.length) - await read(fd, buf, 0, range.length, range.offset) + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) - rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + const videoFilename = file.filename + json[videoFilename] = rangeHashes + }) + }) } - await close(fd) - const videoFilename = getVideoFilename(hlsPlaylist, file) - json[videoFilename] = rangeHashes - } + if (playlist.segmentsSha256Filename) { + await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) + } + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) - await outputJSON(outputPath, json) -} + const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) + await outputJSON(outputPath, json) -function getRangesFromPlaylist (playlistContent: string) { - const ranges: { offset: number, length: number }[] = [] - const lines = playlistContent.split('\n') - const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) + await remove(outputPath) + } - for (const line of lines) { - const captured = regex.exec(line) + return playlist.save() + }) +} - if (captured) { - ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) - } - } +// --------------------------------------------------------------------------- - return ranges +async function buildSha256Segment (segmentPath: string) { + const buf = await readFile(segmentPath) + return sha256(buf) } -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { let timer + let remainingBodyKBLimit = bodyKBLimit logger.info('Importing HLS playlist %s', playlistUrl) - return new Promise(async (res, rej) => { + return new Promise(async (res, rej) => { const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) await ensureDir(tmpDirectory) @@ -138,15 +184,19 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, const subPlaylistUrls = await fetchUniqUrls(playlistUrl) const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) - const fileUrls = uniq(flatten(await Promise.all(subRequests))) + const fileUrls = uniqify(flatten(await Promise.all(subRequests))) logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) for (const fileUrl of fileUrls) { const destPath = join(tmpDirectory, basename(fileUrl)) - const bodyKBLimit = 10 * 1000 * 1000 // 10GB - await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit) + await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) + + const { size } = await stat(destPath) + remainingBodyKBLimit -= (size / 1000) + + logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) } clearTimeout(timer) @@ -167,7 +217,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, } async function fetchUniqUrls (playlistUrl: string) { - const { body } = await doRequest({ uri: playlistUrl }) + const { body } = await doRequest(playlistUrl) if (!body) return [] @@ -179,17 +229,53 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, return `${dirname(playlistUrl)}/${url}` }) - return uniq(urls) + return uniqify(urls) } } // --------------------------------------------------------------------------- +async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { + const content = await readFile(playlistPath, 'utf8') + + const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) + + await writeFile(playlistPath, newContent, 'utf8') +} + +// --------------------------------------------------------------------------- + +function injectQueryToPlaylistUrls (content: string, queryString: string) { + return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) +} + +// --------------------------------------------------------------------------- + export { updateMasterHLSPlaylist, - updateSha256Segments, + updateSha256VODSegments, + buildSha256Segment, downloadPlaylistSegments, - updateStreamingPlaylistsInfohashesIfNeeded + updateStreamingPlaylistsInfohashesIfNeeded, + updatePlaylistAfterFileChange, + injectQueryToPlaylistUrls, + renameVideoFileInPlaylist } // --------------------------------------------------------------------------- + +function getRangesFromPlaylist (playlistContent: string) { + const ranges: { offset: number, length: number }[] = [] + const lines = playlistContent.split('\n') + const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ + + for (const line of lines) { + const captured = regex.exec(line) + + if (captured) { + ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) + } + } + + return ranges +}