From 71e3e879c0616882ee82a0e44f8c2e5ee9698a3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 2 Dec 2022 14:47:21 +0100 Subject: Support reinjecting token in private m3u8 playlist --- server/controllers/object-storage-proxy.ts | 20 ++++++- server/controllers/shared/m3u8-playlist.ts | 14 +++++ server/controllers/static.ts | 39 ++++++++++++++ server/helpers/stream-replacer.ts | 58 ++++++++++++++++++++ server/lib/hls.ts | 9 +++- server/middlewares/validators/static.ts | 11 +++- server/middlewares/validators/users.ts | 2 +- .../object-storage/video-static-file-privacy.ts | 36 ++++++++++++- .../tests/api/videos/video-static-file-privacy.ts | 61 +++++++++++++++++++++- server/tests/shared/checks.ts | 7 +++ server/tests/shared/index.ts | 2 +- server/tests/shared/playlists.ts | 22 -------- server/tests/shared/streaming-playlists.ts | 50 +++++++++++++++++- server/tests/shared/video-playlists.ts | 22 ++++++++ 14 files changed, 321 insertions(+), 32 deletions(-) create mode 100644 server/controllers/shared/m3u8-playlist.ts create mode 100644 server/helpers/stream-replacer.ts delete mode 100644 server/tests/shared/playlists.ts create mode 100644 server/tests/shared/video-playlists.ts (limited to 'server') diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts index 3ce279671..aa853a383 100644 --- a/server/controllers/object-storage-proxy.ts +++ b/server/controllers/object-storage-proxy.ts @@ -1,7 +1,10 @@ import cors from 'cors' import express from 'express' +import { PassThrough, pipeline } from 'stream' import { logger } from '@server/helpers/logger' +import { StreamReplacer } from '@server/helpers/stream-replacer' import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' +import { injectQueryToPlaylistUrls } from '@server/lib/hls' import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' import { asyncMiddleware, @@ -11,6 +14,7 @@ import { optionalAuthenticate } from '@server/middlewares' import { HttpStatusCode } from '@shared/models' +import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' const objectStorageProxyRouter = express.Router() @@ -67,7 +71,20 @@ async function proxifyHLS (req: express.Request, res: express.Response) { rangeHeader: req.header('range') }) - return stream.pipe(res) + const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) + ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req))) + : new PassThrough() + + return pipeline( + stream, + streamReplacer, + res, + err => { + if (!err) return + + handleObjectStorageFailure(res, err) + } + ) } catch (err) { return handleObjectStorageFailure(res, err) } @@ -75,6 +92,7 @@ async function proxifyHLS (req: express.Request, res: express.Response) { function handleObjectStorageFailure (res: express.Response, err: Error) { if (err.name === 'NoSuchKey') { + logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) } diff --git a/server/controllers/shared/m3u8-playlist.ts b/server/controllers/shared/m3u8-playlist.ts new file mode 100644 index 000000000..e2a66efc0 --- /dev/null +++ b/server/controllers/shared/m3u8-playlist.ts @@ -0,0 +1,14 @@ +import express from 'express' + +function doReinjectVideoFileToken (req: express.Request) { + return req.query.videoFileToken && req.query.reinjectVideoFileToken +} + +function buildReinjectVideoFileTokenQuery (req: express.Request) { + return 'videoFileToken=' + req.query.videoFileToken +} + +export { + doReinjectVideoFileToken, + buildReinjectVideoFileTokenQuery +} diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 6ef9154b9..52e48267f 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -1,5 +1,8 @@ import cors from 'cors' import express from 'express' +import { readFile } from 'fs-extra' +import { join } from 'path' +import { injectQueryToPlaylistUrls } from '@server/lib/hls' import { asyncMiddleware, ensureCanAccessPrivateVideoHLSFiles, @@ -7,8 +10,10 @@ import { handleStaticError, optionalAuthenticate } from '@server/middlewares' +import { HttpStatusCode } from '@shared/models' import { CONFIG } from '../initializers/config' import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' +import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' const staticRouter = express.Router() @@ -49,6 +54,12 @@ const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AU ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] : [] +staticRouter.use( + STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8', + ...privateHLSStaticMiddlewares, + asyncMiddleware(servePrivateM3U8) +) + staticRouter.use( STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, ...privateHLSStaticMiddlewares, @@ -74,3 +85,31 @@ staticRouter.use( export { staticRouter } + +// --------------------------------------------------------------------------- + +async function servePrivateM3U8 (req: express.Request, res: express.Response) { + const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8') + + let playlistContent: string + + try { + playlistContent = await readFile(path, 'utf-8') + } catch (err) { + if (err.message.includes('ENOENT')) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'File not found' + }) + } + + throw err + } + + // Inject token in playlist so players that cannot alter the HTTP request can still watch the video + const transformedContent = doReinjectVideoFileToken(req) + ? injectQueryToPlaylistUrls(playlistContent, buildReinjectVideoFileTokenQuery(req)) + : playlistContent + + return res.set('content-type', 'application/vnd.apple.mpegurl').send(transformedContent).end() +} diff --git a/server/helpers/stream-replacer.ts b/server/helpers/stream-replacer.ts new file mode 100644 index 000000000..4babab418 --- /dev/null +++ b/server/helpers/stream-replacer.ts @@ -0,0 +1,58 @@ +import { Transform, TransformCallback } from 'stream' + +// Thanks: https://stackoverflow.com/a/45126242 +class StreamReplacer extends Transform { + private pendingChunk: Buffer + + constructor (private readonly replacer: (line: string) => string) { + super() + } + + _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) { + try { + this.pendingChunk = this.pendingChunk?.length + ? Buffer.concat([ this.pendingChunk, chunk ]) + : chunk + + let index: number + + // As long as we keep finding newlines, keep making slices of the buffer and push them to the + // readable side of the transform stream + while ((index = this.pendingChunk.indexOf('\n')) !== -1) { + // The `end` parameter is non-inclusive, so increase it to include the newline we found + const line = this.pendingChunk.slice(0, ++index) + + // `start` is inclusive, but we are already one char ahead of the newline -> all good + this.pendingChunk = this.pendingChunk.slice(index) + + // We have a single line here! Prepend the string we want + this.push(this.doReplace(line)) + } + + return done() + } catch (err) { + return done(err) + } + } + + _flush (done: TransformCallback) { + // If we have any remaining data in the cache, send it out + if (!this.pendingChunk?.length) return done() + + try { + return done(null, this.doReplace(this.pendingChunk)) + } catch (err) { + return done(err) + } + } + + private doReplace (buffer: Buffer) { + const line = this.replacer(buffer.toString('utf8')) + + return Buffer.from(line, 'utf8') + } +} + +export { + StreamReplacer +} diff --git a/server/lib/hls.ts b/server/lib/hls.ts index a41f1ae48..053b5d326 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -234,13 +234,20 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, // --------------------------------------------------------------------------- +function injectQueryToPlaylistUrls (content: string, queryString: string) { + return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) +} + +// --------------------------------------------------------------------------- + export { updateMasterHLSPlaylist, updateSha256VODSegments, buildSha256Segment, downloadPlaylistSegments, updateStreamingPlaylistsInfohashesIfNeeded, - updatePlaylistAfterFileChange + updatePlaylistAfterFileChange, + injectQueryToPlaylistUrls } // --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts index 13fde6dd1..d3d307787 100644 --- a/server/middlewares/validators/static.ts +++ b/server/middlewares/validators/static.ts @@ -2,7 +2,7 @@ import express from 'express' import { query } from 'express-validator' import LRUCache from 'lru-cache' import { basename, dirname } from 'path' -import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc' +import { exists, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc' import { logger } from '@server/helpers/logger' import { LRU_CACHE } from '@server/initializers/constants' import { VideoModel } from '@server/models/video/video' @@ -60,7 +60,14 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [ ] const ensureCanAccessPrivateVideoHLSFiles = [ - query('videoFileToken').optional().custom(exists), + query('videoFileToken') + .optional() + .custom(exists), + + query('reinjectVideoFileToken') + .optional() + .customSanitizer(toBooleanOrNull) + .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 50327b6ae..64bd9ca70 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -44,7 +44,7 @@ const usersListValidator = [ query('blocked') .optional() .customSanitizer(toBooleanOrNull) - .isBoolean().withMessage('Should be a valid blocked boolena'), + .isBoolean().withMessage('Should be a valid blocked boolean'), (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts index 62edd10ba..71ad35a43 100644 --- a/server/tests/api/object-storage/video-static-file-privacy.ts +++ b/server/tests/api/object-storage/video-static-file-privacy.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { basename } from 'path' -import { expectStartWith } from '@server/tests/shared' +import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared' import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' import { @@ -191,6 +191,20 @@ describe('Object storage for video static file privacy', function () { } }) + it('Should reinject video file token', async function () { + this.timeout(120000) + + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) + + await checkVideoFileTokenReinjection({ + server, + videoUUID: privateVideoUUID, + videoFileToken, + resolutions: [ 240, 720 ], + isLive: false + }) + }) + it('Should update public video to private', async function () { this.timeout(60000) @@ -315,6 +329,26 @@ describe('Object storage for video static file privacy', function () { await checkLiveFiles(permanentLive, permanentLiveId) }) + it('Should reinject video file token in permanent live', async function () { + this.timeout(240000) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) + await server.live.waitUntilPublished({ videoId: permanentLiveId }) + + const video = await server.videos.getWithToken({ id: permanentLiveId }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + + await checkVideoFileTokenReinjection({ + server, + videoUUID: permanentLiveId, + videoFileToken, + resolutions: [ 720 ], + isLive: true + }) + + await stopFfmpeg(ffmpegCommand) + }) + it('Should have created a replay of the normal live with a private static path', async function () { this.timeout(240000) diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts index eaaed5aad..ef0774b41 100644 --- a/server/tests/api/videos/video-static-file-privacy.ts +++ b/server/tests/api/videos/video-static-file-privacy.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { decode } from 'magnet-uri' -import { expectStartWith } from '@server/tests/shared' +import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared' import { getAllFiles, wait } from '@shared/core-utils' import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' import { @@ -248,6 +248,35 @@ describe('Test video static file privacy', function () { await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) }) + it('Should reinject video file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + await waitJobs([ server ]) + + const video = await server.videos.getWithToken({ id: uuid }) + const hls = video.streamingPlaylists[0] + + { + const query = { videoFileToken } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text).to.not.include(videoFileToken) + } + + { + await checkVideoFileTokenReinjection({ + server, + videoUUID: uuid, + videoFileToken, + resolutions: [ 240, 720 ], + isLive: false + }) + } + }) + it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () { this.timeout(120000) @@ -360,6 +389,36 @@ describe('Test video static file privacy', function () { await checkLiveFiles(permanentLive, permanentLiveId) }) + it('Should reinject video file token on permanent live', async function () { + this.timeout(240000) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) + await server.live.waitUntilPublished({ videoId: permanentLiveId }) + + const video = await server.videos.getWithToken({ id: permanentLiveId }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + const hls = video.streamingPlaylists[0] + + { + const query = { videoFileToken } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text).to.not.include(videoFileToken) + } + + { + await checkVideoFileTokenReinjection({ + server, + videoUUID: permanentLiveId, + videoFileToken, + resolutions: [ 720 ], + isLive: true + }) + } + + await stopFfmpeg(ffmpegCommand) + }) + it('Should have created a replay of the normal live with a private static path', async function () { this.timeout(240000) diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index 55ebc6c3e..523d37420 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -23,6 +23,12 @@ function expectNotStartWith (str: string, start: string) { expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false } +function expectEndWith (str: string, end: string) { + expect(str.endsWith(end), `${str} does not end with ${end}`).to.be.true +} + +// --------------------------------------------------------------------------- + async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { const content = await server.servers.getLogContent() @@ -103,6 +109,7 @@ export { testFileExistsOrNot, expectStartWith, expectNotStartWith, + expectEndWith, checkBadStartPagination, checkBadCountPagination, checkBadSortPagination, diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts index 9f7ade53d..963ef8fe6 100644 --- a/server/tests/shared/index.ts +++ b/server/tests/shared/index.ts @@ -6,7 +6,7 @@ export * from './directories' export * from './generate' export * from './live' export * from './notifications' -export * from './playlists' +export * from './video-playlists' export * from './plugins' export * from './requests' export * from './streaming-playlists' diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts deleted file mode 100644 index 8db303fd8..000000000 --- a/server/tests/shared/playlists.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { expect } from 'chai' -import { readdir } from 'fs-extra' -import { PeerTubeServer } from '@shared/server-commands' - -async function checkPlaylistFilesWereRemoved ( - playlistUUID: string, - server: PeerTubeServer, - directories = [ 'thumbnails' ] -) { - for (const directory of directories) { - const directoryPath = server.getDirectoryPath(directory) - - const files = await readdir(directoryPath) - for (const file of files) { - expect(file).to.not.contain(playlistUUID) - } - } -} - -export { - checkPlaylistFilesWereRemoved -} diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts index 824c3dcef..5c62af812 100644 --- a/server/tests/shared/streaming-playlists.ts +++ b/server/tests/shared/streaming-playlists.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { basename } from 'path' +import { basename, dirname, join } from 'path' import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' import { sha256 } from '@shared/extra-utils' import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models' @@ -188,9 +188,55 @@ async function completeCheckHlsPlaylist (options: { } } +async function checkVideoFileTokenReinjection (options: { + server: PeerTubeServer + videoUUID: string + videoFileToken: string + resolutions: number[] + isLive: boolean +}) { + const { server, resolutions, videoFileToken, videoUUID, isLive } = options + + const video = await server.videos.getWithToken({ id: videoUUID }) + const hls = video.streamingPlaylists[0] + + const query = { videoFileToken, reinjectVideoFileToken: 'true' } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + for (let i = 0; i < resolutions.length; i++) { + const resolution = resolutions[i] + + const suffix = isLive + ? i + : `-${resolution}` + + expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}`) + } + + const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text) + expect(resolutionPlaylists).to.have.lengthOf(resolutions.length) + + for (const url of resolutionPlaylists) { + const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 }) + + const extension = isLive + ? '.ts' + : '.mp4' + + expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`) + } +} + +function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) { + return masterContent.match(/^([^.]+\.m3u8.*)/mg) + .map(filename => join(dirname(masterPath), filename)) +} + export { checkSegmentHash, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, - completeCheckHlsPlaylist + completeCheckHlsPlaylist, + extractResolutionPlaylistUrls, + checkVideoFileTokenReinjection } diff --git a/server/tests/shared/video-playlists.ts b/server/tests/shared/video-playlists.ts new file mode 100644 index 000000000..8db303fd8 --- /dev/null +++ b/server/tests/shared/video-playlists.ts @@ -0,0 +1,22 @@ +import { expect } from 'chai' +import { readdir } from 'fs-extra' +import { PeerTubeServer } from '@shared/server-commands' + +async function checkPlaylistFilesWereRemoved ( + playlistUUID: string, + server: PeerTubeServer, + directories = [ 'thumbnails' ] +) { + for (const directory of directories) { + const directoryPath = server.getDirectoryPath(directory) + + const files = await readdir(directoryPath) + for (const file of files) { + expect(file).to.not.contain(playlistUUID) + } + } +} + +export { + checkPlaylistFilesWereRemoved +} -- cgit v1.2.3