X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Fhls.ts;h=1574ff27bac29401a2e5731a9f3804819ca9cf8a;hb=f304a1580b5ce7b6c5f9e25cd3ddc63ca8d8c6a1;hp=212bd095be9cb01fdd7b3fbb3c194c8fefc52cb3;hpb=83903cb65d531a6b6b91715387493ba8312b264d;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 212bd095b..1574ff27b 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,18 +1,19 @@ -import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' +import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' import { flatten, uniq } from 'lodash' import { basename, dirname, join } from 'path' -import { MVideoWithFile } from '@server/types/models' -import { sha256 } from '../helpers/core-utils' +import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' +import { sha256 } from '@shared/extra-utils' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' import { logger } from '../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' import { generateRandomString } from '../helpers/utils' import { CONFIG } from '../initializers/config' -import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants' +import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { getVideoFilePath } from './video-paths' +import { getHlsResolutionPlaylistFilename } from './paths' +import { VideoPathManager } from './video-path-manager' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -22,83 +23,76 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { await sequelizeTypescript.transaction(async t => { const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) - playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) + playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + await playlist.save({ transaction: t }) }) } } -async function updateMasterHLSPlaylist (video: MVideoWithFile) { - const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) +async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) { const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) - const streamingPlaylist = video.getHLSPlaylist() - - for (const file of streamingPlaylist.VideoFiles) { - const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution) - - // If we did not generated a playlist for this resolution, skip - const filePlaylistPath = join(directory, playlistFilename) - if (await pathExists(filePlaylistPath) === false) continue - const videoFilePath = getVideoFilePath(streamingPlaylist, file) + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - const size = await getVideoStreamSize(videoFilePath) + await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { + const size = await getVideoStreamSize(videoFilePath) - const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) - const resolution = `RESOLUTION=${size.width}x${size.height}` + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size.width}x${size.height}` - let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` - if (file.fps) line += ',FRAME-RATE=' + file.fps + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps - const codecs = await Promise.all([ - getVideoStreamCodec(videoFilePath), - getAudioStreamCodec(videoFilePath) - ]) + const codecs = await Promise.all([ + getVideoStreamCodec(videoFilePath), + getAudioStreamCodec(videoFilePath) + ]) - line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` - masterPlaylists.push(line) - masterPlaylists.push(playlistFilename) + masterPlaylists.push(line) + masterPlaylists.push(playlistFilename) + }) } - await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') + await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => { + return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') + }) } -async function updateSha256VODSegments (video: MVideoWithFile) { +async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) { const json: { [filename: string]: { [range: string]: string } } = {} - const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) - const hlsPlaylist = video.getHLSPlaylist() - // For all the resolutions available for this video - for (const file of hlsPlaylist.VideoFiles) { + for (const file of playlist.VideoFiles) { const rangeHashes: { [range: string]: string } = {} + const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - const videoPath = getVideoFilePath(hlsPlaylist, file) - const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - // Maybe the playlist is not generated for this resolution yet - if (!await pathExists(playlistPath)) continue + return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { + const playlistContent = await readFile(resolutionPlaylistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - const playlistContent = await readFile(playlistPath) - const ranges = getRangesFromPlaylist(playlistContent.toString()) + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) - const fd = await open(videoPath, 'r') - for (const range of ranges) { - const buf = Buffer.alloc(range.length) - await read(fd, buf, 0, range.length, range.offset) + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) - rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) - } - await close(fd) - - const videoFilename = file.filename - json[videoFilename] = rangeHashes + const videoFilename = file.filename + json[videoFilename] = rangeHashes + }) + }) } - const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) await outputJSON(outputPath, json) } @@ -107,8 +101,9 @@ async function buildSha256Segment (segmentPath: string) { return sha256(buf) } -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { let timer + let remainingBodyKBLimit = bodyKBLimit logger.info('Importing HLS playlist %s', playlistUrl) @@ -135,8 +130,12 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, for (const fileUrl of fileUrls) { const destPath = join(tmpDirectory, basename(fileUrl)) - const bodyKBLimit = 10 * 1000 * 1000 // 10GB - await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) + await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) + + const { size } = await stat(destPath) + remainingBodyKBLimit -= (size / 1000) + + logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) } clearTimeout(timer)