server/lib/hls.ts

   1 import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
   2 import { flatten, uniq } from 'lodash'
   3 import { basename, dirname, join } from 'path'
   4 import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
   5 import { sha256 } from '@shared/extra-utils'
   6 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
   7 import { logger } from '../helpers/logger'
   8 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
   9 import { generateRandomString } from '../helpers/utils'
  10 import { CONFIG } from '../initializers/config'
  11 import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants'
  12 import { sequelizeTypescript } from '../initializers/database'
  13 import { VideoFileModel } from '../models/video/video-file'
  14 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
  15 import { getHlsResolutionPlaylistFilename } from './paths'
  16 import { VideoPathManager } from './video-path-manager'
  17
  18 async function updateStreamingPlaylistsInfohashesIfNeeded () {
  19   const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
  20
  21   // Use separate SQL queries, because we could have many videos to update
  22   for (const playlist of playlistsToUpdate) {
  23     await sequelizeTypescript.transaction(async t => {
  24       const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
  25
  26       playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
  27       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
  28
  29       await playlist.save({ transaction: t })
  30     })
  31   }
  32 }
  33
  34 async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) {
  35   const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
  36
  37   for (const file of playlist.VideoFiles) {
  38     const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
  39
  40     await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
  41       const size = await getVideoStreamSize(videoFilePath)
  42
  43       const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
  44       const resolution = `RESOLUTION=${size.width}x${size.height}`
  45
  46       let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
  47       if (file.fps) line += ',FRAME-RATE=' + file.fps
  48
  49       const codecs = await Promise.all([
  50         getVideoStreamCodec(videoFilePath),
  51         getAudioStreamCodec(videoFilePath)
  52       ])
  53
  54       line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
  55
  56       masterPlaylists.push(line)
  57       masterPlaylists.push(playlistFilename)
  58     })
  59   }
  60
  61   await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
  62     return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
  63   })
  64 }
  65
  66 async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) {
  67   const json: { [filename: string]: { [range: string]: string } } = {}
  68
  69   // For all the resolutions available for this video
  70   for (const file of playlist.VideoFiles) {
  71     const rangeHashes: { [range: string]: string } = {}
  72     const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
  73
  74     await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
  75
  76       return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
  77         const playlistContent = await readFile(resolutionPlaylistPath)
  78         const ranges = getRangesFromPlaylist(playlistContent.toString())
  79
  80         const fd = await open(videoPath, 'r')
  81         for (const range of ranges) {
  82           const buf = Buffer.alloc(range.length)
  83           await read(fd, buf, 0, range.length, range.offset)
  84
  85           rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
  86         }
  87         await close(fd)
  88
  89         const videoFilename = file.filename
  90         json[videoFilename] = rangeHashes
  91       })
  92     })
  93   }
  94
  95   const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
  96   await outputJSON(outputPath, json)
  97 }
  98
  99 async function buildSha256Segment (segmentPath: string) {
 100   const buf = await readFile(segmentPath)
 101   return sha256(buf)
 102 }
 103
 104 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
 105   let timer
 106   let remainingBodyKBLimit = bodyKBLimit
 107
 108   logger.info('Importing HLS playlist %s', playlistUrl)
 109
 110   return new Promise<void>(async (res, rej) => {
 111     const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
 112
 113     await ensureDir(tmpDirectory)
 114
 115     timer = setTimeout(() => {
 116       deleteTmpDirectory(tmpDirectory)
 117
 118       return rej(new Error('HLS download timeout.'))
 119     }, timeout)
 120
 121     try {
 122       // Fetch master playlist
 123       const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
 124
 125       const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
 126       const fileUrls = uniq(flatten(await Promise.all(subRequests)))
 127
 128       logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
 129
 130       for (const fileUrl of fileUrls) {
 131         const destPath = join(tmpDirectory, basename(fileUrl))
 132
 133         await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
 134
 135         const { size } = await stat(destPath)
 136         remainingBodyKBLimit -= (size / 1000)
 137
 138         logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
 139       }
 140
 141       clearTimeout(timer)
 142
 143       await move(tmpDirectory, destinationDir, { overwrite: true })
 144
 145       return res()
 146     } catch (err) {
 147       deleteTmpDirectory(tmpDirectory)
 148
 149       return rej(err)
 150     }
 151   })
 152
 153   function deleteTmpDirectory (directory: string) {
 154     remove(directory)
 155       .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
 156   }
 157
 158   async function fetchUniqUrls (playlistUrl: string) {
 159     const { body } = await doRequest(playlistUrl)
 160
 161     if (!body) return []
 162
 163     const urls = body.split('\n')
 164       .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
 165       .map(url => {
 166         if (url.startsWith('http://') || url.startsWith('https://')) return url
 167
 168         return `${dirname(playlistUrl)}/${url}`
 169       })
 170
 171     return uniq(urls)
 172   }
 173 }
 174
 175 // ---------------------------------------------------------------------------
 176
 177 export {
 178   updateMasterHLSPlaylist,
 179   updateSha256VODSegments,
 180   buildSha256Segment,
 181   downloadPlaylistSegments,
 182   updateStreamingPlaylistsInfohashesIfNeeded
 183 }
 184
 185 // ---------------------------------------------------------------------------
 186
 187 function getRangesFromPlaylist (playlistContent: string) {
 188   const ranges: { offset: number, length: number }[] = []
 189   const lines = playlistContent.split('\n')
 190   const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
 191
 192   for (const line of lines) {
 193     const captured = regex.exec(line)
 194
 195     if (captured) {
 196       ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
 197     }
 198   }
 199
 200   return ranges
 201 }