X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Fhls.ts;h=04187668c7afd4f2b24b8a50da8dad624c112edd;hb=095e2258043fcff8a79ab082d11edfbd8f13a8e2;hp=10db6c3c345ae0ce7bdabd7f9ccb3338a89fa5b3;hpb=092092969633bbcf6d4891a083ea497a7d5c3154;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 10db6c3c3..04187668c 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,27 +1,48 @@ -import { VideoModel } from '../models/video/video' +import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' +import { flatten, uniq } from 'lodash' import { basename, dirname, join } from 'path' -import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' -import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' -import { getVideoFileSize } from '../helpers/ffmpeg-utils' +import { MVideoWithFile } from '@server/types/models' import { sha256 } from '../helpers/core-utils' -import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import HLSDownloader from 'hlsdownloader' +import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' import { logger } from '../helpers/logger' -import { parse } from 'url' +import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' +import { generateRandomString } from '../helpers/utils' +import { CONFIG } from '../initializers/config' +import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' +import { sequelizeTypescript } from '../initializers/database' +import { VideoFileModel } from '../models/video/video-file' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' +import { getVideoFilePath } from './video-paths' + +async function updateStreamingPlaylistsInfohashesIfNeeded () { + const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() + + // Use separate SQL queries, because we could have many videos to update + for (const playlist of playlistsToUpdate) { + await sequelizeTypescript.transaction(async t => { + const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) -async function updateMasterHLSPlaylist (video: VideoModel) { - const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + await playlist.save({ transaction: t }) + }) + } +} + +async function updateMasterHLSPlaylist (video: MVideoWithFile) { + const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) + const streamingPlaylist = video.getHLSPlaylist() - for (const file of video.VideoFiles) { + for (const file of streamingPlaylist.VideoFiles) { // If we did not generated a playlist for this resolution, skip const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) if (await pathExists(filePlaylistPath) === false) continue - const videoFilePath = video.getVideoFilePath(file) + const videoFilePath = getVideoFilePath(streamingPlaylist, file) - const size = await getVideoFileSize(videoFilePath) + const size = await getVideoStreamSize(videoFilePath) const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) const resolution = `RESOLUTION=${size.width}x${size.height}` @@ -29,6 +50,14 @@ async function updateMasterHLSPlaylist (video: VideoModel) { let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` if (file.fps) line += ',FRAME-RATE=' + file.fps + const videoCodec = await getVideoStreamCodec(videoFilePath) + line += `,CODECS="${videoCodec}` + + const audioCodec = await getAudioStreamCodec(videoFilePath) + if (audioCodec) line += `,${audioCodec}` + + line += '"' + masterPlaylists.push(line) masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) } @@ -36,75 +65,137 @@ async function updateMasterHLSPlaylist (video: VideoModel) { await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') } -async function updateSha256Segments (video: VideoModel) { - const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) - const files = await readdir(directory) - const json: { [filename: string]: string} = {} +async function updateSha256VODSegments (video: MVideoWithFile) { + const json: { [filename: string]: { [range: string]: string } } = {} + + const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) + const hlsPlaylist = video.getHLSPlaylist() + + // For all the resolutions available for this video + for (const file of hlsPlaylist.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} - for (const file of files) { - if (file.endsWith('.ts') === false) continue + const videoPath = getVideoFilePath(hlsPlaylist, file) + const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) - const buffer = await readFile(join(directory, file)) - const filename = basename(file) + // Maybe the playlist is not generated for this resolution yet + if (!await pathExists(playlistPath)) continue - json[filename] = sha256(buffer) + const playlistContent = await readFile(playlistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) + + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) + + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) + + const videoFilename = file.filename + json[videoFilename] = rangeHashes } - const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) await outputJSON(outputPath, json) } +async function buildSha256Segment (segmentPath: string) { + const buf = await readFile(segmentPath) + return sha256(buf) +} + function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { let timer logger.info('Importing HLS playlist %s', playlistUrl) - const params = { - playlistURL: playlistUrl, - destination: CONFIG.STORAGE.TMP_DIR - } - const downloader = new HLSDownloader(params) + return new Promise(async (res, rej) => { + const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) - const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) + await ensureDir(tmpDirectory) - return new Promise(async (res, rej) => { - downloader.startDownload(err => { - clearTimeout(timer) + timer = setTimeout(() => { + deleteTmpDirectory(tmpDirectory) - if (err) { - deleteTmpDirectory(hlsDestinationDir) + return rej(new Error('HLS download timeout.')) + }, timeout) - return rej(err) - } + try { + // Fetch master playlist + const subPlaylistUrls = await fetchUniqUrls(playlistUrl) - move(hlsDestinationDir, destinationDir, { overwrite: true }) - .then(() => res()) - .catch(err => { - deleteTmpDirectory(hlsDestinationDir) + const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) + const fileUrls = uniq(flatten(await Promise.all(subRequests))) - return rej(err) - }) - }) + logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) - timer = setTimeout(() => { - deleteTmpDirectory(hlsDestinationDir) + for (const fileUrl of fileUrls) { + const destPath = join(tmpDirectory, basename(fileUrl)) - return rej(new Error('HLS download timeout.')) - }, timeout) + const bodyKBLimit = 10 * 1000 * 1000 // 10GB + await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit) + } + + clearTimeout(timer) + + await move(tmpDirectory, destinationDir, { overwrite: true }) + + return res() + } catch (err) { + deleteTmpDirectory(tmpDirectory) - function deleteTmpDirectory (directory: string) { - remove(directory) - .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + return rej(err) } }) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + + async function fetchUniqUrls (playlistUrl: string) { + const { body } = await doRequest({ uri: playlistUrl }) + + if (!body) return [] + + const urls = body.split('\n') + .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) + .map(url => { + if (url.startsWith('http://') || url.startsWith('https://')) return url + + return `${dirname(playlistUrl)}/${url}` + }) + + return uniq(urls) + } } // --------------------------------------------------------------------------- export { updateMasterHLSPlaylist, - updateSha256Segments, - downloadPlaylistSegments + updateSha256VODSegments, + buildSha256Segment, + downloadPlaylistSegments, + updateStreamingPlaylistsInfohashesIfNeeded } // --------------------------------------------------------------------------- + +function getRangesFromPlaylist (playlistContent: string) { + const ranges: { offset: number, length: number }[] = [] + const lines = playlistContent.split('\n') + const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ + + for (const line of lines) { + const captured = regex.exec(line) + + if (captured) { + ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) + } + } + + return ranges +}