From 57f879a540551c3b958b0991c8e1e3657a4481d8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 9 Jul 2021 10:21:10 +0200 Subject: [PATCH] Introduce streaming playlists command --- server/tests/api/live/live.ts | 16 ++-- server/tests/api/redundancy/redundancy.ts | 2 +- server/tests/api/videos/video-hls.ts | 20 +++-- shared/extra-utils/server/servers.ts | 5 +- shared/extra-utils/shared/abstract-command.ts | 45 ++++++++-- shared/extra-utils/videos/index.ts | 3 +- .../videos/streaming-playlists-command.ts | 45 ++++++++++ .../extra-utils/videos/streaming-playlists.ts | 76 +++++++++++++++++ .../videos/video-streaming-playlists.ts | 82 ------------------- 9 files changed, 190 insertions(+), 104 deletions(-) create mode 100644 shared/extra-utils/videos/streaming-playlists-command.ts create mode 100644 shared/extra-utils/videos/streaming-playlists.ts delete mode 100644 shared/extra-utils/videos/video-streaming-playlists.ts diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index cb52e4431..0b06df44c 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts @@ -15,7 +15,6 @@ import { doubleFollow, flushAndRunMultipleServers, getMyVideosWithFilter, - getPlaylist, getVideo, getVideosList, getVideosWithFilters, @@ -397,20 +396,27 @@ describe('Test live', function () { // Only finite files are displayed expect(hlsPlaylist.files).to.have.lengthOf(0) - await checkResolutionsInMasterPlaylist(hlsPlaylist.playlistUrl, resolutions) + await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) for (let i = 0; i < resolutions.length; i++) { const segmentNum = 3 const segmentName = `${i}-00000${segmentNum}.ts` await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, resolution: i, segment: segmentNum }) - const res = await getPlaylist(`${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`) - const subPlaylist = res.text + const subPlaylist = await servers[0].streamingPlaylistsCommand.get({ + url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8` + }) expect(subPlaylist).to.contain(segmentName) const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls' - await checkLiveSegmentHash(baseUrlAndPath, video.uuid, segmentName, hlsPlaylist) + await checkLiveSegmentHash({ + server, + baseUrlSegment: baseUrlAndPath, + videoUUID: video.uuid, + segmentName, + hlsPlaylist + }) } } } diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index e4ea99de6..d20cb80f1 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -203,7 +203,7 @@ async function check1PlaylistRedundancies (videoUUID?: string) { const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0] for (const resolution of [ 240, 360, 480, 720 ]) { - await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist) + await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) } const directories = [ diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 3821cfed0..3dd8c9066 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -12,7 +12,6 @@ import { cleanupTests, doubleFollow, flushAndRunMultipleServers, - getPlaylist, getVideo, makeRawRequest, removeVideo, @@ -67,10 +66,9 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, hlsOn } { - await checkResolutionsInMasterPlaylist(hlsPlaylist.playlistUrl, resolutions) + await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) - const res = await getPlaylist(hlsPlaylist.playlistUrl) - const masterPlaylist = res.text + const masterPlaylist = await server.streamingPlaylistsCommand.get({ url: hlsPlaylist.playlistUrl }) for (const resolution of resolutions) { expect(masterPlaylist).to.contain(`${resolution}.m3u8`) @@ -80,9 +78,10 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, hlsOn { for (const resolution of resolutions) { - const res = await getPlaylist(`${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8`) + const subPlaylist = await server.streamingPlaylistsCommand.get({ + url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8` + }) - const subPlaylist = res.text expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`) } } @@ -91,7 +90,14 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, hlsOn const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' for (const resolution of resolutions) { - await checkSegmentHash(baseUrlAndPath, baseUrlAndPath, videoUUID, resolution, hlsPlaylist) + await checkSegmentHash({ + server, + baseUrlPlaylist: baseUrlAndPath, + baseUrlSegment: baseUrlAndPath, + videoUUID, + resolution, + hlsPlaylist + }) } } } diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index 95c876110..6a1dadbcc 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts @@ -26,7 +26,8 @@ import { ImportsCommand, LiveCommand, PlaylistsCommand, - ServicesCommand + ServicesCommand, + StreamingPlaylistsCommand } from '../videos' import { ConfigCommand } from './config-command' import { ContactFormCommand } from './contact-form-command' @@ -117,6 +118,7 @@ interface ServerInfo { playlistsCommand?: PlaylistsCommand historyCommand?: HistoryCommand importsCommand?: ImportsCommand + streamingPlaylistsCommand?: StreamingPlaylistsCommand } function parallelTests () { @@ -350,6 +352,7 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = [] server.playlistsCommand = new PlaylistsCommand(server) server.historyCommand = new HistoryCommand(server) server.importsCommand = new ImportsCommand(server) + server.streamingPlaylistsCommand = new StreamingPlaylistsCommand(server) res(server) }) diff --git a/shared/extra-utils/shared/abstract-command.ts b/shared/extra-utils/shared/abstract-command.ts index 1e07e9469..be368376f 100644 --- a/shared/extra-utils/shared/abstract-command.ts +++ b/shared/extra-utils/shared/abstract-command.ts @@ -16,6 +16,9 @@ export interface OverrideCommandOptions { } interface InternalCommonCommandOptions extends OverrideCommandOptions { + // Default to server.url + url?: string + path: string // If we automatically send the server token if the token is not provided implicitToken: boolean @@ -27,6 +30,7 @@ interface InternalGetCommandOptions extends InternalCommonCommandOptions { contentType?: string accept?: string redirects?: number + range?: string } abstract class AbstractCommand { @@ -55,6 +59,22 @@ abstract class AbstractCommand { return unwrapText(this.getRequest(options)) } + protected getRawRequest (options: Omit) { + const { url, range } = options + const { host, protocol, pathname } = new URL(url) + + return this.getRequest({ + ...options, + + token: this.buildCommonRequestToken(options), + defaultExpectedStatus: this.buildStatusCodeExpected(options), + + url: `${protocol}//${host}`, + path: pathname, + range + }) + } + protected getRequest (options: InternalGetCommandOptions) { const { redirects, query, contentType, accept } = options @@ -127,20 +147,31 @@ abstract class AbstractCommand { } private buildCommonRequestOptions (options: InternalCommonCommandOptions) { - const { token, expectedStatus, defaultExpectedStatus, path } = options + const { path } = options + + return { + url: this.server.url, + path, + + token: this.buildCommonRequestToken(options), + statusCodeExpected: this.buildStatusCodeExpected(options) + } + } + + private buildCommonRequestToken (options: Pick) { + const { token } = options const fallbackToken = options.implicitToken ? this.server.accessToken : undefined - return { - url: this.server.url, - path, + return token !== undefined ? token : fallbackToken + } - token: token !== undefined ? token : fallbackToken, + private buildStatusCodeExpected (options: Pick) { + const { expectedStatus, defaultExpectedStatus } = options - statusCodeExpected: expectedStatus ?? this.expectedStatus ?? defaultExpectedStatus - } + return expectedStatus ?? this.expectedStatus ?? defaultExpectedStatus } } diff --git a/shared/extra-utils/videos/index.ts b/shared/extra-utils/videos/index.ts index 372cf7a90..f87ae8eea 100644 --- a/shared/extra-utils/videos/index.ts +++ b/shared/extra-utils/videos/index.ts @@ -9,7 +9,8 @@ export * from './live' export * from './playlists-command' export * from './playlists' export * from './services-command' +export * from './streaming-playlists-command' +export * from './streaming-playlists' export * from './video-channels' export * from './video-comments' -export * from './video-streaming-playlists' export * from './videos' diff --git a/shared/extra-utils/videos/streaming-playlists-command.ts b/shared/extra-utils/videos/streaming-playlists-command.ts new file mode 100644 index 000000000..4caec7137 --- /dev/null +++ b/shared/extra-utils/videos/streaming-playlists-command.ts @@ -0,0 +1,45 @@ + +import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' +import { unwrapBody, unwrapText } from '../requests' +import { AbstractCommand, OverrideCommandOptions } from '../shared' + +export class StreamingPlaylistsCommand extends AbstractCommand { + + get (options: OverrideCommandOptions & { + url: string + }) { + return unwrapText(this.getRawRequest({ + ...options, + + url: options.url, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + getSegment (options: OverrideCommandOptions & { + url: string + range?: string + }) { + return unwrapText(this.getRawRequest({ + ...options, + + url: options.url, + range: options.range, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200, + })) + } + + getSegmentSha256 (options: OverrideCommandOptions & { + url: string + }) { + return unwrapBody<{ [ id: string ]: string }>(this.getRawRequest({ + ...options, + + url: options.url, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } +} diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts new file mode 100644 index 000000000..0324c739a --- /dev/null +++ b/shared/extra-utils/videos/streaming-playlists.ts @@ -0,0 +1,76 @@ +import { expect } from 'chai' +import { sha256 } from '@server/helpers/core-utils' +import { HttpStatusCode } from '@shared/core-utils' +import { VideoStreamingPlaylist } from '@shared/models' +import { ServerInfo } from '../server' + +async function checkSegmentHash (options: { + server: ServerInfo + baseUrlPlaylist: string + baseUrlSegment: string + videoUUID: string + resolution: number + hlsPlaylist: VideoStreamingPlaylist +}) { + const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options + const command = server.streamingPlaylistsCommand + + const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8` }) + + const videoName = `${videoUUID}-${resolution}-fragmented.mp4` + + const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) + + const length = parseInt(matches[1], 10) + const offset = parseInt(matches[2], 10) + const range = `${offset}-${offset + length - 1}` + + const segmentBody = await command.getSegment({ + url: `${baseUrlSegment}/${videoUUID}/${videoName}`, + expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, + range: `bytes=${range}` + }) + + const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) + expect(sha256(segmentBody)).to.equal(shaBody[videoName][range]) +} + +async function checkLiveSegmentHash (options: { + server: ServerInfo + baseUrlSegment: string + videoUUID: string + segmentName: string + hlsPlaylist: VideoStreamingPlaylist +}) { + const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options + const command = server.streamingPlaylistsCommand + + const segmentBody = await command.getSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) + const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) + + expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) +} + +async function checkResolutionsInMasterPlaylist (options: { + server: ServerInfo + playlistUrl: string + resolutions: number[] +}) { + const { server, playlistUrl, resolutions } = options + + const masterPlaylist = await server.streamingPlaylistsCommand.get({ url: playlistUrl }) + + for (const resolution of resolutions) { + const reg = new RegExp( + '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"' + ) + + expect(masterPlaylist).to.match(reg) + } +} + +export { + checkSegmentHash, + checkLiveSegmentHash, + checkResolutionsInMasterPlaylist +} diff --git a/shared/extra-utils/videos/video-streaming-playlists.ts b/shared/extra-utils/videos/video-streaming-playlists.ts deleted file mode 100644 index 99c2e1880..000000000 --- a/shared/extra-utils/videos/video-streaming-playlists.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { makeRawRequest } from '../requests/requests' -import { sha256 } from '../../../server/helpers/core-utils' -import { VideoStreamingPlaylist } from '../../models/videos/video-streaming-playlist.model' -import { expect } from 'chai' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' - -function getPlaylist (url: string, statusCodeExpected = HttpStatusCode.OK_200) { - return makeRawRequest(url, statusCodeExpected) -} - -function getSegment (url: string, statusCodeExpected = HttpStatusCode.OK_200, range?: string) { - return makeRawRequest(url, statusCodeExpected, range) -} - -function getSegmentSha256 (url: string, statusCodeExpected = HttpStatusCode.OK_200) { - return makeRawRequest(url, statusCodeExpected) -} - -async function checkSegmentHash ( - baseUrlPlaylist: string, - baseUrlSegment: string, - videoUUID: string, - resolution: number, - hlsPlaylist: VideoStreamingPlaylist -) { - const res = await getPlaylist(`${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8`) - const playlist = res.text - - const videoName = `${videoUUID}-${resolution}-fragmented.mp4` - - const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) - - const length = parseInt(matches[1], 10) - const offset = parseInt(matches[2], 10) - const range = `${offset}-${offset + length - 1}` - - const res2 = await getSegment(`${baseUrlSegment}/${videoUUID}/${videoName}`, HttpStatusCode.PARTIAL_CONTENT_206, `bytes=${range}`) - - const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) - - const sha256Server = resSha.body[videoName][range] - expect(sha256(res2.body)).to.equal(sha256Server) -} - -async function checkLiveSegmentHash ( - baseUrlSegment: string, - videoUUID: string, - segmentName: string, - hlsPlaylist: VideoStreamingPlaylist -) { - const res2 = await getSegment(`${baseUrlSegment}/${videoUUID}/${segmentName}`) - - const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url) - - const sha256Server = resSha.body[segmentName] - expect(sha256(res2.body)).to.equal(sha256Server) -} - -async function checkResolutionsInMasterPlaylist (playlistUrl: string, resolutions: number[]) { - const res = await getPlaylist(playlistUrl) - - const masterPlaylist = res.text - - for (const resolution of resolutions) { - const reg = new RegExp( - '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + ',(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"' - ) - - expect(masterPlaylist).to.match(reg) - } -} - -// --------------------------------------------------------------------------- - -export { - getPlaylist, - getSegment, - checkResolutionsInMasterPlaylist, - getSegmentSha256, - checkLiveSegmentHash, - checkSegmentHash -} -- 2.41.0