server/tests/shared/streaming-playlists.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { basename, dirname, join } from 'path'
   5 import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
   6 import { sha256 } from '@shared/extra-utils'
   7 import { HttpStatusCode, VideoPrivacy, VideoResolution, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
   8 import { makeRawRequest, PeerTubeServer } from '@shared/server-commands'
   9 import { expectStartWith } from './checks'
  10 import { hlsInfohashExist } from './tracker'
  11 import { checkWebTorrentWorks } from './webtorrent'
  12
  13 async function checkSegmentHash (options: {
  14   server: PeerTubeServer
  15   baseUrlPlaylist: string
  16   baseUrlSegment: string
  17   resolution: number
  18   hlsPlaylist: VideoStreamingPlaylist
  19   token?: string
  20 }) {
  21   const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist, token } = options
  22   const command = server.streamingPlaylists
  23
  24   const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
  25   const videoName = basename(file.fileUrl)
  26
  27   const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8`, token })
  28
  29   const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
  30
  31   const length = parseInt(matches[1], 10)
  32   const offset = parseInt(matches[2], 10)
  33   const range = `${offset}-${offset + length - 1}`
  34
  35   const segmentBody = await command.getFragmentedSegment({
  36     url: `${baseUrlSegment}/${videoName}`,
  37     expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206,
  38     range: `bytes=${range}`,
  39     token
  40   })
  41
  42   const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, token })
  43   expect(sha256(segmentBody)).to.equal(shaBody[videoName][range], `Invalid sha256 result for ${videoName} range ${range}`)
  44 }
  45
  46 // ---------------------------------------------------------------------------
  47
  48 async function checkLiveSegmentHash (options: {
  49   server: PeerTubeServer
  50   baseUrlSegment: string
  51   videoUUID: string
  52   segmentName: string
  53   hlsPlaylist: VideoStreamingPlaylist
  54   withRetry?: boolean
  55 }) {
  56   const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist, withRetry = false } = options
  57   const command = server.streamingPlaylists
  58
  59   const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}`, withRetry })
  60   const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry })
  61
  62   expect(sha256(segmentBody)).to.equal(shaBody[segmentName])
  63 }
  64
  65 // ---------------------------------------------------------------------------
  66
  67 async function checkResolutionsInMasterPlaylist (options: {
  68   server: PeerTubeServer
  69   playlistUrl: string
  70   resolutions: number[]
  71   token?: string
  72   transcoded?: boolean // default true
  73   withRetry?: boolean // default false
  74 }) {
  75   const { server, playlistUrl, resolutions, token, withRetry = false, transcoded = true } = options
  76
  77   const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl, token, withRetry })
  78
  79   for (const resolution of resolutions) {
  80     const base = '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution
  81
  82     if (resolution === VideoResolution.H_NOVIDEO) {
  83       expect(masterPlaylist).to.match(new RegExp(`${base},CODECS="mp4a.40.2"`))
  84     } else if (transcoded) {
  85       expect(masterPlaylist).to.match(new RegExp(`${base},(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"`))
  86     } else {
  87       expect(masterPlaylist).to.match(new RegExp(`${base}`))
  88     }
  89   }
  90
  91   const playlistsLength = masterPlaylist.split('\n').filter(line => line.startsWith('#EXT-X-STREAM-INF:BANDWIDTH='))
  92   expect(playlistsLength).to.have.lengthOf(resolutions.length)
  93 }
  94
  95 async function completeCheckHlsPlaylist (options: {
  96   servers: PeerTubeServer[]
  97   videoUUID: string
  98   hlsOnly: boolean
  99
 100   resolutions?: number[]
 101   objectStorageBaseUrl?: string
 102 }) {
 103   const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
 104
 105   const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
 106
 107   for (const server of options.servers) {
 108     const videoDetails = await server.videos.getWithToken({ id: videoUUID })
 109     const requiresAuth = videoDetails.privacy.id === VideoPrivacy.PRIVATE || videoDetails.privacy.id === VideoPrivacy.INTERNAL
 110
 111     const privatePath = requiresAuth
 112       ? 'private/'
 113       : ''
 114     const token = requiresAuth
 115       ? server.accessToken
 116       : undefined
 117
 118     const baseUrl = `http://${videoDetails.account.host}`
 119
 120     expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
 121
 122     const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
 123     expect(hlsPlaylist).to.not.be.undefined
 124
 125     const hlsFiles = hlsPlaylist.files
 126     expect(hlsFiles).to.have.lengthOf(resolutions.length)
 127
 128     if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
 129     else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
 130
 131     // Check JSON files
 132     for (const resolution of resolutions) {
 133       const file = hlsFiles.find(f => f.resolution.id === resolution)
 134       expect(file).to.not.be.undefined
 135
 136       if (file.resolution.id === VideoResolution.H_NOVIDEO) {
 137         expect(file.resolution.label).to.equal('Audio')
 138       } else {
 139         expect(file.resolution.label).to.equal(resolution + 'p')
 140       }
 141
 142       expect(file.magnetUri).to.have.lengthOf.above(2)
 143       await checkWebTorrentWorks(file.magnetUri)
 144
 145       {
 146         const nameReg = `${uuidRegex}-${file.resolution.id}`
 147
 148         expect(file.torrentUrl).to.match(new RegExp(`${server.url}/lazy-static/torrents/${nameReg}-hls.torrent`))
 149
 150         if (objectStorageBaseUrl && requiresAuth) {
 151           // eslint-disable-next-line max-len
 152           expect(file.fileUrl).to.match(new RegExp(`${server.url}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoDetails.uuid}/${nameReg}-fragmented.mp4`))
 153         } else if (objectStorageBaseUrl) {
 154           expectStartWith(file.fileUrl, objectStorageBaseUrl)
 155         } else {
 156           expect(file.fileUrl).to.match(
 157             new RegExp(`${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoDetails.uuid}/${nameReg}-fragmented.mp4`)
 158           )
 159         }
 160       }
 161
 162       {
 163         await Promise.all([
 164           makeRawRequest({ url: file.torrentUrl, token, expectedStatus: HttpStatusCode.OK_200 }),
 165           makeRawRequest({ url: file.torrentDownloadUrl, token, expectedStatus: HttpStatusCode.OK_200 }),
 166           makeRawRequest({ url: file.metadataUrl, token, expectedStatus: HttpStatusCode.OK_200 }),
 167           makeRawRequest({ url: file.fileUrl, token, expectedStatus: HttpStatusCode.OK_200 }),
 168
 169           makeRawRequest({
 170             url: file.fileDownloadUrl,
 171             token,
 172             expectedStatus: objectStorageBaseUrl
 173               ? HttpStatusCode.FOUND_302
 174               : HttpStatusCode.OK_200
 175           })
 176         ])
 177       }
 178     }
 179
 180     // Check master playlist
 181     {
 182       await checkResolutionsInMasterPlaylist({ server, token, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
 183
 184       const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl, token })
 185
 186       let i = 0
 187       for (const resolution of resolutions) {
 188         expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
 189         expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
 190
 191         const url = 'http://' + videoDetails.account.host
 192         await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
 193
 194         i++
 195       }
 196     }
 197
 198     // Check resolution playlists
 199     {
 200       for (const resolution of resolutions) {
 201         const file = hlsFiles.find(f => f.resolution.id === resolution)
 202         const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
 203
 204         let url: string
 205         if (objectStorageBaseUrl && requiresAuth) {
 206           url = `${baseUrl}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoUUID}/${playlistName}`
 207         } else if (objectStorageBaseUrl) {
 208           url = `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
 209         } else {
 210           url = `${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoUUID}/${playlistName}`
 211         }
 212
 213         const subPlaylist = await server.streamingPlaylists.get({ url, token })
 214
 215         expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
 216         expect(subPlaylist).to.contain(basename(file.fileUrl))
 217       }
 218     }
 219
 220     {
 221       let baseUrlAndPath: string
 222       if (objectStorageBaseUrl && requiresAuth) {
 223         baseUrlAndPath = `${baseUrl}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoUUID}`
 224       } else if (objectStorageBaseUrl) {
 225         baseUrlAndPath = `${objectStorageBaseUrl}hls/${videoUUID}`
 226       } else {
 227         baseUrlAndPath = `${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoUUID}`
 228       }
 229
 230       for (const resolution of resolutions) {
 231         await checkSegmentHash({
 232           server,
 233           token,
 234           baseUrlPlaylist: baseUrlAndPath,
 235           baseUrlSegment: baseUrlAndPath,
 236           resolution,
 237           hlsPlaylist
 238         })
 239       }
 240     }
 241   }
 242 }
 243
 244 async function checkVideoFileTokenReinjection (options: {
 245   server: PeerTubeServer
 246   videoUUID: string
 247   videoFileToken: string
 248   resolutions: number[]
 249   isLive: boolean
 250 }) {
 251   const { server, resolutions, videoFileToken, videoUUID, isLive } = options
 252
 253   const video = await server.videos.getWithToken({ id: videoUUID })
 254   const hls = video.streamingPlaylists[0]
 255
 256   const query = { videoFileToken, reinjectVideoFileToken: 'true' }
 257   const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 })
 258
 259   for (let i = 0; i < resolutions.length; i++) {
 260     const resolution = resolutions[i]
 261
 262     const suffix = isLive
 263       ? i
 264       : `-${resolution}`
 265
 266     expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}&reinjectVideoFileToken=true`)
 267   }
 268
 269   const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text)
 270   expect(resolutionPlaylists).to.have.lengthOf(resolutions.length)
 271
 272   for (const url of resolutionPlaylists) {
 273     const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 })
 274
 275     const extension = isLive
 276       ? '.ts'
 277       : '.mp4'
 278
 279     expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`)
 280     expect(text).not.to.contain(`reinjectVideoFileToken=true`)
 281   }
 282 }
 283
 284 function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) {
 285   return masterContent.match(/^([^.]+\.m3u8.*)/mg)
 286     .map(filename => join(dirname(masterPath), filename))
 287 }
 288
 289 export {
 290   checkSegmentHash,
 291   checkLiveSegmentHash,
 292   checkResolutionsInMasterPlaylist,
 293   completeCheckHlsPlaylist,
 294   extractResolutionPlaylistUrls,
 295   checkVideoFileTokenReinjection
 296 }