From 4c280004ce62bf11ddb091854c28f1e1d54a54d6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:08:19 +0100 Subject: Use a single file instead of segments for HLS --- server/helpers/ffmpeg-utils.ts | 12 ++- server/helpers/requests.ts | 2 +- server/helpers/utils.ts | 1 - server/lib/activitypub/actor.ts | 4 +- server/lib/hls.ts | 136 +++++++++++++++++------- server/lib/video-transcoding.ts | 5 +- server/models/video/video-streaming-playlist.ts | 4 + server/tests/api/redundancy/redundancy.ts | 22 ++-- server/tests/api/videos/video-hls.ts | 16 +-- 9 files changed, 132 insertions(+), 70 deletions(-) (limited to 'server') diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 5ad8ed48e..133b1b03b 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -122,7 +122,9 @@ type TranscodeOptions = { resolution: VideoResolution isPortraitMode?: boolean - generateHlsPlaylist?: boolean + hlsPlaylist?: { + videoFilename: string + } } function transcode (options: TranscodeOptions) { @@ -161,14 +163,16 @@ function transcode (options: TranscodeOptions) { command = command.withFPS(fps) } - if (options.generateHlsPlaylist) { - const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts` + if (options.hlsPlaylist) { + const videoPath = `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` command = command.outputOption('-hls_time 4') .outputOption('-hls_list_size 0') .outputOption('-hls_playlist_type vod') - .outputOption('-hls_segment_filename ' + segmentFilename) + .outputOption('-hls_segment_filename ' + videoPath) + .outputOption('-hls_segment_type fmp4') .outputOption('-f hls') + .outputOption('-hls_flags single_file') } command diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 3fc776f1a..5c6dc5e19 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -7,7 +7,7 @@ import { join } from 'path' function doRequest ( requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } -): Bluebird<{ response: request.RequestResponse, body: any }> { +): Bluebird<{ response: request.RequestResponse, body: T }> { if (requestOptions.activityPub === true) { if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 3c3406e38..cb0e823c5 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -7,7 +7,6 @@ import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' import { remove } from 'fs-extra' import * as memoizee from 'memoizee' -import { isArray } from './custom-validators/misc' function deleteFileAsync (path: string) { remove(path) diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 8215840da..a3f379b76 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -355,10 +355,10 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe logger.info('Fetching remote actor %s.', actorUrl) - const requestResult = await doRequest(options) + const requestResult = await doRequest(options) normalizeActor(requestResult.body) - const actorJSON: ActivityPubActor = requestResult.body + const actorJSON = requestResult.body if (isActorObjectValid(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON }) return { result: undefined, statusCode: requestResult.response.statusCode } diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 10db6c3c3..3575981f4 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,13 +1,14 @@ import { VideoModel } from '../models/video/video' -import { basename, dirname, join } from 'path' -import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers' -import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra' +import { basename, join, dirname } from 'path' +import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers' +import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' import { getVideoFileSize } from '../helpers/ffmpeg-utils' import { sha256 } from '../helpers/core-utils' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import HLSDownloader from 'hlsdownloader' import { logger } from '../helpers/logger' -import { parse } from 'url' +import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' +import { generateRandomString } from '../helpers/utils' +import { flatten, uniq } from 'lodash' async function updateMasterHLSPlaylist (video: VideoModel) { const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) @@ -37,66 +38,119 @@ async function updateMasterHLSPlaylist (video: VideoModel) { } async function updateSha256Segments (video: VideoModel) { - const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) - const files = await readdir(directory) - const json: { [filename: string]: string} = {} + const json: { [filename: string]: { [range: string]: string } } = {} + + const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid) + + // For all the resolutions available for this video + for (const file of video.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + + const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) + const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) - for (const file of files) { - if (file.endsWith('.ts') === false) continue + // Maybe the playlist is not generated for this resolution yet + if (!await pathExists(playlistPath)) continue - const buffer = await readFile(join(directory, file)) - const filename = basename(file) + const playlistContent = await readFile(playlistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) - json[filename] = sha256(buffer) + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await read(fd, buf, 0, range.length, range.offset) + + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await close(fd) + + const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) + json[videoFilename] = rangeHashes } - const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) await outputJSON(outputPath, json) } -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { - let timer +function getRangesFromPlaylist (playlistContent: string) { + const ranges: { offset: number, length: number }[] = [] + const lines = playlistContent.split('\n') + const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ - logger.info('Importing HLS playlist %s', playlistUrl) + for (const line of lines) { + const captured = regex.exec(line) - const params = { - playlistURL: playlistUrl, - destination: CONFIG.STORAGE.TMP_DIR + if (captured) { + ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) + } } - const downloader = new HLSDownloader(params) - - const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname)) - return new Promise(async (res, rej) => { - downloader.startDownload(err => { - clearTimeout(timer) + return ranges +} - if (err) { - deleteTmpDirectory(hlsDestinationDir) +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) { + let timer - return rej(err) - } + logger.info('Importing HLS playlist %s', playlistUrl) - move(hlsDestinationDir, destinationDir, { overwrite: true }) - .then(() => res()) - .catch(err => { - deleteTmpDirectory(hlsDestinationDir) + return new Promise(async (res, rej) => { + const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) - return rej(err) - }) - }) + await ensureDir(tmpDirectory) timer = setTimeout(() => { - deleteTmpDirectory(hlsDestinationDir) + deleteTmpDirectory(tmpDirectory) return rej(new Error('HLS download timeout.')) }, timeout) - function deleteTmpDirectory (directory: string) { - remove(directory) - .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + try { + // Fetch master playlist + const subPlaylistUrls = await fetchUniqUrls(playlistUrl) + + const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) + const fileUrls = uniq(flatten(await Promise.all(subRequests))) + + logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) + + for (const fileUrl of fileUrls) { + const destPath = join(tmpDirectory, basename(fileUrl)) + + await doRequestAndSaveToFile({ uri: fileUrl }, destPath) + } + + clearTimeout(timer) + + await move(tmpDirectory, destinationDir, { overwrite: true }) + + return res() + } catch (err) { + deleteTmpDirectory(tmpDirectory) + + return rej(err) } }) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + + async function fetchUniqUrls (playlistUrl: string) { + const { body } = await doRequest({ uri: playlistUrl }) + + if (!body) return [] + + const urls = body.split('\n') + .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) + .map(url => { + if (url.startsWith('http://') || url.startsWith('https://')) return url + + return `${dirname(playlistUrl)}/${url}` + }) + + return uniq(urls) + } } // --------------------------------------------------------------------------- diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index 608badfef..086b860a2 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -100,7 +100,10 @@ async function generateHlsPlaylist (video: VideoModel, resolution: VideoResoluti outputPath, resolution, isPortraitMode, - generateHlsPlaylist: true + + hlsPlaylist: { + videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) + } } await transcode(transcodeOptions) diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index bce537781..bf6f7b0c4 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -125,6 +125,10 @@ export class VideoStreamingPlaylistModel extends Model f === `${resolution}_000.ts`)).to.not.be.undefined - expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined + const filename = `${videoUUID}-${resolution}-fragmented.mp4` + + expect(files.find(f => f === filename)).to.not.be.undefined } } } diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 71d863b12..a1214bad1 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -4,13 +4,12 @@ import * as chai from 'chai' import 'mocha' import { checkDirectoryIsEmpty, + checkSegmentHash, checkTmpIsEmpty, doubleFollow, flushAndRunMultipleServers, flushTests, getPlaylist, - getSegment, - getSegmentSha256, getVideo, killallServers, removeVideo, @@ -22,7 +21,6 @@ import { } from '../../../../shared/utils' import { VideoDetails } from '../../../../shared/models/videos' import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type' -import { sha256 } from '../../../helpers/core-utils' import { join } from 'path' const expect = chai.expect @@ -56,19 +54,15 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) { const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`) const subPlaylist = res2.text - expect(subPlaylist).to.contain(resolution + '_000.ts') + expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`) } } { - for (const resolution of resolutions) { - - const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`) + const baseUrl = 'http://localhost:9001/static/playlists/hls' - const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) - - const sha256Server = resSha.body[ resolution + '_000.ts' ] - expect(sha256(res2.body)).to.equal(sha256Server) + for (const resolution of resolutions) { + await checkSegmentHash(baseUrl, baseUrl, videoUUID, resolution, hlsPlaylist) } } } -- cgit v1.2.3