aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-12-02 14:47:21 +0100
committerChocobozzz <me@florianbigard.com>2022-12-02 15:25:20 +0100
commit71e3e879c0616882ee82a0e44f8c2e5ee9698a3e (patch)
tree14452d26d240eb6d44178b76fc2dabda4cfc9428
parent04509c43254dc232c61681ac4bb98e09fd126115 (diff)
downloadPeerTube-71e3e879c0616882ee82a0e44f8c2e5ee9698a3e.tar.gz
PeerTube-71e3e879c0616882ee82a0e44f8c2e5ee9698a3e.tar.zst
PeerTube-71e3e879c0616882ee82a0e44f8c2e5ee9698a3e.zip
Support reinjecting token in private m3u8 playlist
-rw-r--r--client/e2e/src/suites-all/private-videos.e2e-spec.ts15
-rw-r--r--client/e2e/src/utils/urls.ts4
-rw-r--r--client/src/assets/player/shared/manager-options/hls-options-builder.ts1
-rw-r--r--client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts76
-rw-r--r--client/src/assets/player/shared/p2p-media-loader/segment-validator.ts3
-rw-r--r--client/src/assets/player/types/peertube-videojs-typings.ts1
-rw-r--r--server/controllers/object-storage-proxy.ts20
-rw-r--r--server/controllers/shared/m3u8-playlist.ts14
-rw-r--r--server/controllers/static.ts39
-rw-r--r--server/helpers/stream-replacer.ts58
-rw-r--r--server/lib/hls.ts9
-rw-r--r--server/middlewares/validators/static.ts11
-rw-r--r--server/middlewares/validators/users.ts2
-rw-r--r--server/tests/api/object-storage/video-static-file-privacy.ts36
-rw-r--r--server/tests/api/videos/video-static-file-privacy.ts61
-rw-r--r--server/tests/shared/checks.ts7
-rw-r--r--server/tests/shared/index.ts2
-rw-r--r--server/tests/shared/streaming-playlists.ts50
-rw-r--r--server/tests/shared/video-playlists.ts (renamed from server/tests/shared/playlists.ts)0
-rw-r--r--shared/core-utils/common/url.ts9
-rw-r--r--shared/server-commands/videos/streaming-playlists-command.ts10
-rw-r--r--support/doc/api/openapi.yaml9
22 files changed, 390 insertions, 47 deletions
diff --git a/client/e2e/src/suites-all/private-videos.e2e-spec.ts b/client/e2e/src/suites-all/private-videos.e2e-spec.ts
index db3554659..a25208bb3 100644
--- a/client/e2e/src/suites-all/private-videos.e2e-spec.ts
+++ b/client/e2e/src/suites-all/private-videos.e2e-spec.ts
@@ -15,6 +15,7 @@ describe('Private videos all workflow', () => {
15 let playerPage: PlayerPage 15 let playerPage: PlayerPage
16 16
17 const internalVideoName = 'Internal E2E test' 17 const internalVideoName = 'Internal E2E test'
18 const internalHLSOnlyVideoName = 'Internal E2E test - HLS only'
18 19
19 beforeEach(async () => { 20 beforeEach(async () => {
20 videoWatchPage = new VideoWatchPage(isMobileDevice(), isSafari()) 21 videoWatchPage = new VideoWatchPage(isMobileDevice(), isSafari())
@@ -44,6 +45,13 @@ describe('Private videos all workflow', () => {
44 await checkCorrectlyPlay(playerPage) 45 await checkCorrectlyPlay(playerPage)
45 }) 46 })
46 47
48 it('Should play an internal HLS only video', async () => {
49 await go(FIXTURE_URLS.INTERNAL_HLS_ONLY_VIDEO)
50
51 await videoWatchPage.waitWatchVideoName(internalHLSOnlyVideoName)
52 await checkCorrectlyPlay(playerPage)
53 })
54
47 it('Should play an internal WebTorrent video in embed', async () => { 55 it('Should play an internal WebTorrent video in embed', async () => {
48 await go(FIXTURE_URLS.INTERNAL_EMBED_WEBTORRENT_VIDEO) 56 await go(FIXTURE_URLS.INTERNAL_EMBED_WEBTORRENT_VIDEO)
49 57
@@ -57,4 +65,11 @@ describe('Private videos all workflow', () => {
57 await videoWatchPage.waitEmbedForDisplayed() 65 await videoWatchPage.waitEmbedForDisplayed()
58 await checkCorrectlyPlay(playerPage) 66 await checkCorrectlyPlay(playerPage)
59 }) 67 })
68
69 it('Should play an internal HLS only video in embed', async () => {
70 await go(FIXTURE_URLS.INTERNAL_EMBED_HLS_ONLY_VIDEO)
71
72 await videoWatchPage.waitEmbedForDisplayed()
73 await checkCorrectlyPlay(playerPage)
74 })
60}) 75})
diff --git a/client/e2e/src/utils/urls.ts b/client/e2e/src/utils/urls.ts
index f91d9a048..cc0bdfbff 100644
--- a/client/e2e/src/utils/urls.ts
+++ b/client/e2e/src/utils/urls.ts
@@ -1,9 +1,13 @@
1const FIXTURE_URLS = { 1const FIXTURE_URLS = {
2 INTERNAL_WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/pwfz7NizSdPD4mJcbbmNwa?mode=webtorrent&start=0', 2 INTERNAL_WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/pwfz7NizSdPD4mJcbbmNwa?mode=webtorrent&start=0',
3 INTERNAL_HLS_VIDEO: 'https://peertube2.cpy.re/w/pwfz7NizSdPD4mJcbbmNwa?start=0', 3 INTERNAL_HLS_VIDEO: 'https://peertube2.cpy.re/w/pwfz7NizSdPD4mJcbbmNwa?start=0',
4
4 INTERNAL_EMBED_WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/videos/embed/pwfz7NizSdPD4mJcbbmNwa?mode=webtorrent&start=0', 5 INTERNAL_EMBED_WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/videos/embed/pwfz7NizSdPD4mJcbbmNwa?mode=webtorrent&start=0',
5 INTERNAL_EMBED_HLS_VIDEO: 'https://peertube2.cpy.re/videos/embed/pwfz7NizSdPD4mJcbbmNwa?start=0', 6 INTERNAL_EMBED_HLS_VIDEO: 'https://peertube2.cpy.re/videos/embed/pwfz7NizSdPD4mJcbbmNwa?start=0',
6 7
8 INTERNAL_HLS_ONLY_VIDEO: 'https://peertube2.cpy.re/w/tKQmHcqdYZRdCszLUiWM3V?start=0',
9 INTERNAL_EMBED_HLS_ONLY_VIDEO: 'https://peertube2.cpy.re/videos/embed/tKQmHcqdYZRdCszLUiWM3V?start=0',
10
7 WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/122d093a-1ede-43bd-bd34-59d2931ffc5e', 11 WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/122d093a-1ede-43bd-bd34-59d2931ffc5e',
8 12
9 HLS_EMBED: 'https://peertube2.cpy.re/videos/embed/969bf103-7818-43b5-94a0-de159e13de50', 13 HLS_EMBED: 'https://peertube2.cpy.re/videos/embed/969bf103-7818-43b5-94a0-de159e13de50',
diff --git a/client/src/assets/player/shared/manager-options/hls-options-builder.ts b/client/src/assets/player/shared/manager-options/hls-options-builder.ts
index 497a97436..63e9fa8c8 100644
--- a/client/src/assets/player/shared/manager-options/hls-options-builder.ts
+++ b/client/src/assets/player/shared/manager-options/hls-options-builder.ts
@@ -32,6 +32,7 @@ export class HLSOptionsBuilder {
32 32
33 const p2pMediaLoader: P2PMediaLoaderPluginOptions = { 33 const p2pMediaLoader: P2PMediaLoaderPluginOptions = {
34 requiresAuth: commonOptions.requiresAuth, 34 requiresAuth: commonOptions.requiresAuth,
35 videoFileToken: commonOptions.videoFileToken,
35 36
36 redundancyUrlManager, 37 redundancyUrlManager,
37 type: 'application/x-mpegURL', 38 type: 'application/x-mpegURL',
diff --git a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts
index b608ee3e2..e6f525fea 100644
--- a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts
+++ b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts
@@ -3,7 +3,7 @@ import videojs from 'video.js'
3import { Events, Segment } from '@peertube/p2p-media-loader-core' 3import { Events, Segment } from '@peertube/p2p-media-loader-core'
4import { Engine, initHlsJsPlayer, initVideoJsContribHlsJsPlayer } from '@peertube/p2p-media-loader-hlsjs' 4import { Engine, initHlsJsPlayer, initVideoJsContribHlsJsPlayer } from '@peertube/p2p-media-loader-hlsjs'
5import { logger } from '@root-helpers/logger' 5import { logger } from '@root-helpers/logger'
6import { timeToInt } from '@shared/core-utils' 6import { addQueryParams, timeToInt } from '@shared/core-utils'
7import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo } from '../../types' 7import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo } from '../../types'
8import { registerConfigPlugin, registerSourceHandler } from './hls-plugin' 8import { registerConfigPlugin, registerSourceHandler } from './hls-plugin'
9 9
@@ -39,46 +39,37 @@ class P2pMediaLoaderPlugin extends Plugin {
39 super(player) 39 super(player)
40 40
41 this.options = options 41 this.options = options
42 this.startTime = timeToInt(options.startTime)
42 43
43 // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080 44 // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080
44 if (!(videojs as any).Html5Hlsjs) { 45 if (!(videojs as any).Html5Hlsjs) {
45 logger.warn('HLS.js does not seem to be supported. Try to fallback to built in HLS.') 46 if (player.canPlayType('application/vnd.apple.mpegurl')) {
46 47 this.fallbackToBuiltInIOS()
47 let message: string 48 return
48 if (!player.canPlayType('application/vnd.apple.mpegurl')) {
49 message = 'Cannot fallback to built-in HLS'
50 } else if (options.requiresAuth) {
51 message = 'Video requires auth which is not compatible to build-in HLS player'
52 } 49 }
53 50
54 if (message) { 51 const message = 'HLS.js does not seem to be supported. Cannot fallback to built-in HLS'
55 logger.warn(message) 52 logger.warn(message)
56 53
57 const error: MediaError = { 54 const error: MediaError = {
58 code: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED, 55 code: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED,
59 message, 56 message,
60 MEDIA_ERR_ABORTED: MediaError.MEDIA_ERR_ABORTED, 57 MEDIA_ERR_ABORTED: MediaError.MEDIA_ERR_ABORTED,
61 MEDIA_ERR_DECODE: MediaError.MEDIA_ERR_DECODE, 58 MEDIA_ERR_DECODE: MediaError.MEDIA_ERR_DECODE,
62 MEDIA_ERR_NETWORK: MediaError.MEDIA_ERR_NETWORK, 59 MEDIA_ERR_NETWORK: MediaError.MEDIA_ERR_NETWORK,
63 MEDIA_ERR_SRC_NOT_SUPPORTED: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED 60 MEDIA_ERR_SRC_NOT_SUPPORTED: MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED
64 }
65
66 player.ready(() => player.error(error))
67 return
68 } 61 }
69 62
70 // Workaround to force video.js to not re create a video element 63 player.ready(() => player.error(error))
71 (this.player as any).playerElIngest_ = this.player.el().parentNode 64 return
72 } else {
73 // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080
74 (videojs as any).Html5Hlsjs.addHook('beforeinitialize', (videojsPlayer: any, hlsjs: any) => {
75 this.hlsjs = hlsjs
76 })
77
78 initVideoJsContribHlsJsPlayer(player)
79 } 65 }
80 66
81 this.startTime = timeToInt(options.startTime) 67 // FIXME: typings https://github.com/Microsoft/TypeScript/issues/14080
68 (videojs as any).Html5Hlsjs.addHook('beforeinitialize', (_videojsPlayer: any, hlsjs: any) => {
69 this.hlsjs = hlsjs
70 })
71
72 initVideoJsContribHlsJsPlayer(player)
82 73
83 player.src({ 74 player.src({
84 type: options.type, 75 type: options.type,
@@ -88,9 +79,7 @@ class P2pMediaLoaderPlugin extends Plugin {
88 player.ready(() => { 79 player.ready(() => {
89 this.initializeCore() 80 this.initializeCore()
90 81
91 if ((videojs as any).Html5Hlsjs) { 82 this.initializePlugin()
92 this.initializePlugin()
93 }
94 }) 83 })
95 } 84 }
96 85
@@ -199,6 +188,25 @@ class P2pMediaLoaderPlugin extends Plugin {
199 private arraySum (data: number[]) { 188 private arraySum (data: number[]) {
200 return data.reduce((a: number, b: number) => a + b, 0) 189 return data.reduce((a: number, b: number) => a + b, 0)
201 } 190 }
191
192 private fallbackToBuiltInIOS () {
193 logger.info('HLS.js does not seem to be supported. Fallback to built-in HLS.');
194
195 // Workaround to force video.js to not re create a video element
196 (this.player as any).playerElIngest_ = this.player.el().parentNode
197
198 this.player.src({
199 type: this.options.type,
200 src: addQueryParams(this.options.src, {
201 videoFileToken: this.options.videoFileToken(),
202 reinjectVideoFileToken: 'true'
203 })
204 })
205
206 this.player.ready(() => {
207 this.initializeCore()
208 })
209 }
202} 210}
203 211
204videojs.registerPlugin('p2pMediaLoader', P2pMediaLoaderPlugin) 212videojs.registerPlugin('p2pMediaLoader', P2pMediaLoaderPlugin)
diff --git a/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts b/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts
index a7ee91950..3c76d63f7 100644
--- a/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts
+++ b/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts
@@ -2,6 +2,7 @@ import { basename } from 'path'
2import { Segment } from '@peertube/p2p-media-loader-core' 2import { Segment } from '@peertube/p2p-media-loader-core'
3import { logger } from '@root-helpers/logger' 3import { logger } from '@root-helpers/logger'
4import { wait } from '@root-helpers/utils' 4import { wait } from '@root-helpers/utils'
5import { removeQueryParams } from '@shared/core-utils'
5import { isSameOrigin } from '../common' 6import { isSameOrigin } from '../common'
6 7
7type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } } 8type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } }
@@ -24,7 +25,7 @@ function segmentValidatorFactory (options: {
24 // Wait for hash generation from the server 25 // Wait for hash generation from the server
25 if (isLive) await wait(1000) 26 if (isLive) await wait(1000)
26 27
27 const filename = basename(segment.url) 28 const filename = basename(removeQueryParams(segment.url))
28 29
29 const segmentValue = (await segmentsJSON)[filename] 30 const segmentValue = (await segmentsJSON)[filename]
30 31
diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts
index 3d9d5270e..c60154f3b 100644
--- a/client/src/assets/player/types/peertube-videojs-typings.ts
+++ b/client/src/assets/player/types/peertube-videojs-typings.ts
@@ -168,6 +168,7 @@ type P2PMediaLoaderPluginOptions = {
168 loader: P2PMediaLoader 168 loader: P2PMediaLoader
169 169
170 requiresAuth: boolean 170 requiresAuth: boolean
171 videoFileToken: () => string
171} 172}
172 173
173export type P2PMediaLoader = { 174export type P2PMediaLoader = {
diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts
index 3ce279671..aa853a383 100644
--- a/server/controllers/object-storage-proxy.ts
+++ b/server/controllers/object-storage-proxy.ts
@@ -1,7 +1,10 @@
1import cors from 'cors' 1import cors from 'cors'
2import express from 'express' 2import express from 'express'
3import { PassThrough, pipeline } from 'stream'
3import { logger } from '@server/helpers/logger' 4import { logger } from '@server/helpers/logger'
5import { StreamReplacer } from '@server/helpers/stream-replacer'
4import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' 6import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
7import { injectQueryToPlaylistUrls } from '@server/lib/hls'
5import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' 8import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
6import { 9import {
7 asyncMiddleware, 10 asyncMiddleware,
@@ -11,6 +14,7 @@ import {
11 optionalAuthenticate 14 optionalAuthenticate
12} from '@server/middlewares' 15} from '@server/middlewares'
13import { HttpStatusCode } from '@shared/models' 16import { HttpStatusCode } from '@shared/models'
17import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist'
14 18
15const objectStorageProxyRouter = express.Router() 19const objectStorageProxyRouter = express.Router()
16 20
@@ -67,7 +71,20 @@ async function proxifyHLS (req: express.Request, res: express.Response) {
67 rangeHeader: req.header('range') 71 rangeHeader: req.header('range')
68 }) 72 })
69 73
70 return stream.pipe(res) 74 const streamReplacer = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req)
75 ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req)))
76 : new PassThrough()
77
78 return pipeline(
79 stream,
80 streamReplacer,
81 res,
82 err => {
83 if (!err) return
84
85 handleObjectStorageFailure(res, err)
86 }
87 )
71 } catch (err) { 88 } catch (err) {
72 return handleObjectStorageFailure(res, err) 89 return handleObjectStorageFailure(res, err)
73 } 90 }
@@ -75,6 +92,7 @@ async function proxifyHLS (req: express.Request, res: express.Response) {
75 92
76function handleObjectStorageFailure (res: express.Response, err: Error) { 93function handleObjectStorageFailure (res: express.Response, err: Error) {
77 if (err.name === 'NoSuchKey') { 94 if (err.name === 'NoSuchKey') {
95 logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
78 return res.sendStatus(HttpStatusCode.NOT_FOUND_404) 96 return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
79 } 97 }
80 98
diff --git a/server/controllers/shared/m3u8-playlist.ts b/server/controllers/shared/m3u8-playlist.ts
new file mode 100644
index 000000000..e2a66efc0
--- /dev/null
+++ b/server/controllers/shared/m3u8-playlist.ts
@@ -0,0 +1,14 @@
1import express from 'express'
2
3function doReinjectVideoFileToken (req: express.Request) {
4 return req.query.videoFileToken && req.query.reinjectVideoFileToken
5}
6
7function buildReinjectVideoFileTokenQuery (req: express.Request) {
8 return 'videoFileToken=' + req.query.videoFileToken
9}
10
11export {
12 doReinjectVideoFileToken,
13 buildReinjectVideoFileTokenQuery
14}
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 6ef9154b9..52e48267f 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,5 +1,8 @@
1import cors from 'cors' 1import cors from 'cors'
2import express from 'express' 2import express from 'express'
3import { readFile } from 'fs-extra'
4import { join } from 'path'
5import { injectQueryToPlaylistUrls } from '@server/lib/hls'
3import { 6import {
4 asyncMiddleware, 7 asyncMiddleware,
5 ensureCanAccessPrivateVideoHLSFiles, 8 ensureCanAccessPrivateVideoHLSFiles,
@@ -7,8 +10,10 @@ import {
7 handleStaticError, 10 handleStaticError,
8 optionalAuthenticate 11 optionalAuthenticate
9} from '@server/middlewares' 12} from '@server/middlewares'
13import { HttpStatusCode } from '@shared/models'
10import { CONFIG } from '../initializers/config' 14import { CONFIG } from '../initializers/config'
11import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' 15import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
16import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist'
12 17
13const staticRouter = express.Router() 18const staticRouter = express.Router()
14 19
@@ -50,6 +55,12 @@ const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AU
50 : [] 55 : []
51 56
52staticRouter.use( 57staticRouter.use(
58 STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8',
59 ...privateHLSStaticMiddlewares,
60 asyncMiddleware(servePrivateM3U8)
61)
62
63staticRouter.use(
53 STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, 64 STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
54 ...privateHLSStaticMiddlewares, 65 ...privateHLSStaticMiddlewares,
55 express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), 66 express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
@@ -74,3 +85,31 @@ staticRouter.use(
74export { 85export {
75 staticRouter 86 staticRouter
76} 87}
88
89// ---------------------------------------------------------------------------
90
91async function servePrivateM3U8 (req: express.Request, res: express.Response) {
92 const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8')
93
94 let playlistContent: string
95
96 try {
97 playlistContent = await readFile(path, 'utf-8')
98 } catch (err) {
99 if (err.message.includes('ENOENT')) {
100 return res.fail({
101 status: HttpStatusCode.NOT_FOUND_404,
102 message: 'File not found'
103 })
104 }
105
106 throw err
107 }
108
109 // Inject token in playlist so players that cannot alter the HTTP request can still watch the video
110 const transformedContent = doReinjectVideoFileToken(req)
111 ? injectQueryToPlaylistUrls(playlistContent, buildReinjectVideoFileTokenQuery(req))
112 : playlistContent
113
114 return res.set('content-type', 'application/vnd.apple.mpegurl').send(transformedContent).end()
115}
diff --git a/server/helpers/stream-replacer.ts b/server/helpers/stream-replacer.ts
new file mode 100644
index 000000000..4babab418
--- /dev/null
+++ b/server/helpers/stream-replacer.ts
@@ -0,0 +1,58 @@
1import { Transform, TransformCallback } from 'stream'
2
3// Thanks: https://stackoverflow.com/a/45126242
4class StreamReplacer extends Transform {
5 private pendingChunk: Buffer
6
7 constructor (private readonly replacer: (line: string) => string) {
8 super()
9 }
10
11 _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) {
12 try {
13 this.pendingChunk = this.pendingChunk?.length
14 ? Buffer.concat([ this.pendingChunk, chunk ])
15 : chunk
16
17 let index: number
18
19 // As long as we keep finding newlines, keep making slices of the buffer and push them to the
20 // readable side of the transform stream
21 while ((index = this.pendingChunk.indexOf('\n')) !== -1) {
22 // The `end` parameter is non-inclusive, so increase it to include the newline we found
23 const line = this.pendingChunk.slice(0, ++index)
24
25 // `start` is inclusive, but we are already one char ahead of the newline -> all good
26 this.pendingChunk = this.pendingChunk.slice(index)
27
28 // We have a single line here! Prepend the string we want
29 this.push(this.doReplace(line))
30 }
31
32 return done()
33 } catch (err) {
34 return done(err)
35 }
36 }
37
38 _flush (done: TransformCallback) {
39 // If we have any remaining data in the cache, send it out
40 if (!this.pendingChunk?.length) return done()
41
42 try {
43 return done(null, this.doReplace(this.pendingChunk))
44 } catch (err) {
45 return done(err)
46 }
47 }
48
49 private doReplace (buffer: Buffer) {
50 const line = this.replacer(buffer.toString('utf8'))
51
52 return Buffer.from(line, 'utf8')
53 }
54}
55
56export {
57 StreamReplacer
58}
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index a41f1ae48..053b5d326 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -234,13 +234,20 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
234 234
235// --------------------------------------------------------------------------- 235// ---------------------------------------------------------------------------
236 236
237function injectQueryToPlaylistUrls (content: string, queryString: string) {
238 return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
239}
240
241// ---------------------------------------------------------------------------
242
237export { 243export {
238 updateMasterHLSPlaylist, 244 updateMasterHLSPlaylist,
239 updateSha256VODSegments, 245 updateSha256VODSegments,
240 buildSha256Segment, 246 buildSha256Segment,
241 downloadPlaylistSegments, 247 downloadPlaylistSegments,
242 updateStreamingPlaylistsInfohashesIfNeeded, 248 updateStreamingPlaylistsInfohashesIfNeeded,
243 updatePlaylistAfterFileChange 249 updatePlaylistAfterFileChange,
250 injectQueryToPlaylistUrls
244} 251}
245 252
246// --------------------------------------------------------------------------- 253// ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts
index 13fde6dd1..d3d307787 100644
--- a/server/middlewares/validators/static.ts
+++ b/server/middlewares/validators/static.ts
@@ -2,7 +2,7 @@ import express from 'express'
2import { query } from 'express-validator' 2import { query } from 'express-validator'
3import LRUCache from 'lru-cache' 3import LRUCache from 'lru-cache'
4import { basename, dirname } from 'path' 4import { basename, dirname } from 'path'
5import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc' 5import { exists, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc'
6import { logger } from '@server/helpers/logger' 6import { logger } from '@server/helpers/logger'
7import { LRU_CACHE } from '@server/initializers/constants' 7import { LRU_CACHE } from '@server/initializers/constants'
8import { VideoModel } from '@server/models/video/video' 8import { VideoModel } from '@server/models/video/video'
@@ -60,7 +60,14 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [
60] 60]
61 61
62const ensureCanAccessPrivateVideoHLSFiles = [ 62const ensureCanAccessPrivateVideoHLSFiles = [
63 query('videoFileToken').optional().custom(exists), 63 query('videoFileToken')
64 .optional()
65 .custom(exists),
66
67 query('reinjectVideoFileToken')
68 .optional()
69 .customSanitizer(toBooleanOrNull)
70 .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'),
64 71
65 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 72 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
66 if (areValidationErrors(req, res)) return 73 if (areValidationErrors(req, res)) return
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index 50327b6ae..64bd9ca70 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -44,7 +44,7 @@ const usersListValidator = [
44 query('blocked') 44 query('blocked')
45 .optional() 45 .optional()
46 .customSanitizer(toBooleanOrNull) 46 .customSanitizer(toBooleanOrNull)
47 .isBoolean().withMessage('Should be a valid blocked boolena'), 47 .isBoolean().withMessage('Should be a valid blocked boolean'),
48 48
49 (req: express.Request, res: express.Response, next: express.NextFunction) => { 49 (req: express.Request, res: express.Response, next: express.NextFunction) => {
50 if (areValidationErrors(req, res)) return 50 if (areValidationErrors(req, res)) return
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts
index 62edd10ba..71ad35a43 100644
--- a/server/tests/api/object-storage/video-static-file-privacy.ts
+++ b/server/tests/api/object-storage/video-static-file-privacy.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { basename } from 'path' 4import { basename } from 'path'
5import { expectStartWith } from '@server/tests/shared' 5import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared'
6import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' 6import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils'
7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' 7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
8import { 8import {
@@ -191,6 +191,20 @@ describe('Object storage for video static file privacy', function () {
191 } 191 }
192 }) 192 })
193 193
194 it('Should reinject video file token', async function () {
195 this.timeout(120000)
196
197 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID })
198
199 await checkVideoFileTokenReinjection({
200 server,
201 videoUUID: privateVideoUUID,
202 videoFileToken,
203 resolutions: [ 240, 720 ],
204 isLive: false
205 })
206 })
207
194 it('Should update public video to private', async function () { 208 it('Should update public video to private', async function () {
195 this.timeout(60000) 209 this.timeout(60000)
196 210
@@ -315,6 +329,26 @@ describe('Object storage for video static file privacy', function () {
315 await checkLiveFiles(permanentLive, permanentLiveId) 329 await checkLiveFiles(permanentLive, permanentLiveId)
316 }) 330 })
317 331
332 it('Should reinject video file token in permanent live', async function () {
333 this.timeout(240000)
334
335 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey })
336 await server.live.waitUntilPublished({ videoId: permanentLiveId })
337
338 const video = await server.videos.getWithToken({ id: permanentLiveId })
339 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
340
341 await checkVideoFileTokenReinjection({
342 server,
343 videoUUID: permanentLiveId,
344 videoFileToken,
345 resolutions: [ 720 ],
346 isLive: true
347 })
348
349 await stopFfmpeg(ffmpegCommand)
350 })
351
318 it('Should have created a replay of the normal live with a private static path', async function () { 352 it('Should have created a replay of the normal live with a private static path', async function () {
319 this.timeout(240000) 353 this.timeout(240000)
320 354
diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts
index eaaed5aad..ef0774b41 100644
--- a/server/tests/api/videos/video-static-file-privacy.ts
+++ b/server/tests/api/videos/video-static-file-privacy.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { decode } from 'magnet-uri' 4import { decode } from 'magnet-uri'
5import { expectStartWith } from '@server/tests/shared' 5import { checkVideoFileTokenReinjection, expectStartWith } from '@server/tests/shared'
6import { getAllFiles, wait } from '@shared/core-utils' 6import { getAllFiles, wait } from '@shared/core-utils'
7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' 7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
8import { 8import {
@@ -248,6 +248,35 @@ describe('Test video static file privacy', function () {
248 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) 248 await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
249 }) 249 })
250 250
251 it('Should reinject video file token', async function () {
252 this.timeout(120000)
253
254 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
255
256 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
257 await waitJobs([ server ])
258
259 const video = await server.videos.getWithToken({ id: uuid })
260 const hls = video.streamingPlaylists[0]
261
262 {
263 const query = { videoFileToken }
264 const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
265
266 expect(text).to.not.include(videoFileToken)
267 }
268
269 {
270 await checkVideoFileTokenReinjection({
271 server,
272 videoUUID: uuid,
273 videoFileToken,
274 resolutions: [ 240, 720 ],
275 isLive: false
276 })
277 }
278 })
279
251 it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () { 280 it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () {
252 this.timeout(120000) 281 this.timeout(120000)
253 282
@@ -360,6 +389,36 @@ describe('Test video static file privacy', function () {
360 await checkLiveFiles(permanentLive, permanentLiveId) 389 await checkLiveFiles(permanentLive, permanentLiveId)
361 }) 390 })
362 391
392 it('Should reinject video file token on permanent live', async function () {
393 this.timeout(240000)
394
395 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey })
396 await server.live.waitUntilPublished({ videoId: permanentLiveId })
397
398 const video = await server.videos.getWithToken({ id: permanentLiveId })
399 const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
400 const hls = video.streamingPlaylists[0]
401
402 {
403 const query = { videoFileToken }
404 const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
405
406 expect(text).to.not.include(videoFileToken)
407 }
408
409 {
410 await checkVideoFileTokenReinjection({
411 server,
412 videoUUID: permanentLiveId,
413 videoFileToken,
414 resolutions: [ 720 ],
415 isLive: true
416 })
417 }
418
419 await stopFfmpeg(ffmpegCommand)
420 })
421
363 it('Should have created a replay of the normal live with a private static path', async function () { 422 it('Should have created a replay of the normal live with a private static path', async function () {
364 this.timeout(240000) 423 this.timeout(240000)
365 424
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts
index 55ebc6c3e..523d37420 100644
--- a/server/tests/shared/checks.ts
+++ b/server/tests/shared/checks.ts
@@ -23,6 +23,12 @@ function expectNotStartWith (str: string, start: string) {
23 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false 23 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false
24} 24}
25 25
26function expectEndWith (str: string, end: string) {
27 expect(str.endsWith(end), `${str} does not end with ${end}`).to.be.true
28}
29
30// ---------------------------------------------------------------------------
31
26async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { 32async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
27 const content = await server.servers.getLogContent() 33 const content = await server.servers.getLogContent()
28 34
@@ -103,6 +109,7 @@ export {
103 testFileExistsOrNot, 109 testFileExistsOrNot,
104 expectStartWith, 110 expectStartWith,
105 expectNotStartWith, 111 expectNotStartWith,
112 expectEndWith,
106 checkBadStartPagination, 113 checkBadStartPagination,
107 checkBadCountPagination, 114 checkBadCountPagination,
108 checkBadSortPagination, 115 checkBadSortPagination,
diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts
index 9f7ade53d..963ef8fe6 100644
--- a/server/tests/shared/index.ts
+++ b/server/tests/shared/index.ts
@@ -6,7 +6,7 @@ export * from './directories'
6export * from './generate' 6export * from './generate'
7export * from './live' 7export * from './live'
8export * from './notifications' 8export * from './notifications'
9export * from './playlists' 9export * from './video-playlists'
10export * from './plugins' 10export * from './plugins'
11export * from './requests' 11export * from './requests'
12export * from './streaming-playlists' 12export * from './streaming-playlists'
diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts
index 824c3dcef..5c62af812 100644
--- a/server/tests/shared/streaming-playlists.ts
+++ b/server/tests/shared/streaming-playlists.ts
@@ -1,7 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { basename } from 'path' 4import { basename, dirname, join } from 'path'
5import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' 5import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
6import { sha256 } from '@shared/extra-utils' 6import { sha256 } from '@shared/extra-utils'
7import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models' 7import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
@@ -188,9 +188,55 @@ async function completeCheckHlsPlaylist (options: {
188 } 188 }
189} 189}
190 190
191async function checkVideoFileTokenReinjection (options: {
192 server: PeerTubeServer
193 videoUUID: string
194 videoFileToken: string
195 resolutions: number[]
196 isLive: boolean
197}) {
198 const { server, resolutions, videoFileToken, videoUUID, isLive } = options
199
200 const video = await server.videos.getWithToken({ id: videoUUID })
201 const hls = video.streamingPlaylists[0]
202
203 const query = { videoFileToken, reinjectVideoFileToken: 'true' }
204 const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
205
206 for (let i = 0; i < resolutions.length; i++) {
207 const resolution = resolutions[i]
208
209 const suffix = isLive
210 ? i
211 : `-${resolution}`
212
213 expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}`)
214 }
215
216 const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text)
217 expect(resolutionPlaylists).to.have.lengthOf(resolutions.length)
218
219 for (const url of resolutionPlaylists) {
220 const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 })
221
222 const extension = isLive
223 ? '.ts'
224 : '.mp4'
225
226 expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`)
227 }
228}
229
230function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) {
231 return masterContent.match(/^([^.]+\.m3u8.*)/mg)
232 .map(filename => join(dirname(masterPath), filename))
233}
234
191export { 235export {
192 checkSegmentHash, 236 checkSegmentHash,
193 checkLiveSegmentHash, 237 checkLiveSegmentHash,
194 checkResolutionsInMasterPlaylist, 238 checkResolutionsInMasterPlaylist,
195 completeCheckHlsPlaylist 239 completeCheckHlsPlaylist,
240 extractResolutionPlaylistUrls,
241 checkVideoFileTokenReinjection
196} 242}
diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/video-playlists.ts
index 8db303fd8..8db303fd8 100644
--- a/server/tests/shared/playlists.ts
+++ b/server/tests/shared/video-playlists.ts
diff --git a/shared/core-utils/common/url.ts b/shared/core-utils/common/url.ts
index d1c399f7b..33fc5ee3a 100644
--- a/shared/core-utils/common/url.ts
+++ b/shared/core-utils/common/url.ts
@@ -11,6 +11,14 @@ function addQueryParams (url: string, params: { [ id: string ]: string }) {
11 return objUrl.toString() 11 return objUrl.toString()
12} 12}
13 13
14function removeQueryParams (url: string) {
15 const objUrl = new URL(url)
16
17 objUrl.searchParams.forEach((_v, k) => objUrl.searchParams.delete(k))
18
19 return objUrl.toString()
20}
21
14function buildPlaylistLink (playlist: Pick<VideoPlaylist, 'shortUUID'>, base?: string) { 22function buildPlaylistLink (playlist: Pick<VideoPlaylist, 'shortUUID'>, base?: string) {
15 return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist) 23 return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist)
16} 24}
@@ -114,6 +122,7 @@ function decoratePlaylistLink (options: {
114 122
115export { 123export {
116 addQueryParams, 124 addQueryParams,
125 removeQueryParams,
117 126
118 buildPlaylistLink, 127 buildPlaylistLink,
119 buildVideoLink, 128 buildVideoLink,
diff --git a/shared/server-commands/videos/streaming-playlists-command.ts b/shared/server-commands/videos/streaming-playlists-command.ts
index 25e446e72..26ab2735f 100644
--- a/shared/server-commands/videos/streaming-playlists-command.ts
+++ b/shared/server-commands/videos/streaming-playlists-command.ts
@@ -7,16 +7,24 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
7 7
8 async get (options: OverrideCommandOptions & { 8 async get (options: OverrideCommandOptions & {
9 url: string 9 url: string
10
11 videoFileToken?: string
12 reinjectVideoFileToken?: boolean
13
10 withRetry?: boolean // default false 14 withRetry?: boolean // default false
11 currentRetry?: number 15 currentRetry?: number
12 }) { 16 }) {
13 const { withRetry, currentRetry = 1 } = options 17 const { videoFileToken, reinjectVideoFileToken, withRetry, currentRetry = 1 } = options
14 18
15 try { 19 try {
16 const result = await unwrapTextOrDecode(this.getRawRequest({ 20 const result = await unwrapTextOrDecode(this.getRawRequest({
17 ...options, 21 ...options,
18 22
19 url: options.url, 23 url: options.url,
24 query: {
25 videoFileToken,
26 reinjectVideoFileToken
27 },
20 implicitToken: false, 28 implicitToken: false,
21 defaultExpectedStatus: HttpStatusCode.OK_200 29 defaultExpectedStatus: HttpStatusCode.OK_200
22 })) 30 }))
diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml
index 2062f2e3a..c2f9d424e 100644
--- a/support/doc/api/openapi.yaml
+++ b/support/doc/api/openapi.yaml
@@ -408,6 +408,7 @@ paths:
408 parameters: 408 parameters:
409 - $ref: '#/components/parameters/staticFilename' 409 - $ref: '#/components/parameters/staticFilename'
410 - $ref: '#/components/parameters/videoFileToken' 410 - $ref: '#/components/parameters/videoFileToken'
411 - $ref: '#/components/parameters/reinjectVideoFileToken'
411 security: 412 security:
412 - OAuth2: [] 413 - OAuth2: []
413 responses: 414 responses:
@@ -5711,7 +5712,13 @@ components:
5711 description: Video file token [generated](#operation/requestVideoToken) by PeerTube so you don't need to provide an OAuth token in the request header. 5712 description: Video file token [generated](#operation/requestVideoToken) by PeerTube so you don't need to provide an OAuth token in the request header.
5712 schema: 5713 schema:
5713 type: string 5714 type: string
5714 5715 reinjectVideoFileToken:
5716 name: reinjectVideoFileToken
5717 in: query
5718 required: false
5719 description: Ask the server to reinject videoFileToken in URLs in m3u8 playlist
5720 schema:
5721 type: boolean
5715 5722
5716 securitySchemes: 5723 securitySchemes:
5717 OAuth2: 5724 OAuth2: