server/lib/hls.ts

   1 import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
   2 import { flatten, uniq } from 'lodash'
   3 import { basename, dirname, join } from 'path'
   4 import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
   5 import { sha256 } from '@shared/extra-utils'
   6 import { VideoStorage } from '@shared/models'
   7 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
   8 import { logger } from '../helpers/logger'
   9 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
  10 import { generateRandomString } from '../helpers/utils'
  11 import { CONFIG } from '../initializers/config'
  12 import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants'
  13 import { sequelizeTypescript } from '../initializers/database'
  14 import { VideoFileModel } from '../models/video/video-file'
  15 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
  16 import { storeHLSFile } from './object-storage'
  17 import { getHlsResolutionPlaylistFilename } from './paths'
  18 import { VideoPathManager } from './video-path-manager'
  19
  20 async function updateStreamingPlaylistsInfohashesIfNeeded () {
  21   const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
  22
  23   // Use separate SQL queries, because we could have many videos to update
  24   for (const playlist of playlistsToUpdate) {
  25     await sequelizeTypescript.transaction(async t => {
  26       const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
  27
  28       playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
  29       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
  30
  31       await playlist.save({ transaction: t })
  32     })
  33   }
  34 }
  35
  36 async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) {
  37   const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
  38
  39   for (const file of playlist.VideoFiles) {
  40     const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
  41
  42     await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
  43       const size = await getVideoStreamDimensionsInfo(videoFilePath)
  44
  45       const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
  46       const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
  47
  48       let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
  49       if (file.fps) line += ',FRAME-RATE=' + file.fps
  50
  51       const codecs = await Promise.all([
  52         getVideoStreamCodec(videoFilePath),
  53         getAudioStreamCodec(videoFilePath)
  54       ])
  55
  56       line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
  57
  58       masterPlaylists.push(line)
  59       masterPlaylists.push(playlistFilename)
  60     })
  61   }
  62
  63   await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, async masterPlaylistPath => {
  64     await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
  65
  66     if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
  67       await storeHLSFile(playlist, playlist.playlistFilename, masterPlaylistPath)
  68     }
  69   })
  70 }
  71
  72 async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) {
  73   const json: { [filename: string]: { [range: string]: string } } = {}
  74
  75   // For all the resolutions available for this video
  76   for (const file of playlist.VideoFiles) {
  77     const rangeHashes: { [range: string]: string } = {}
  78     const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
  79
  80     await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
  81
  82       return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
  83         const playlistContent = await readFile(resolutionPlaylistPath)
  84         const ranges = getRangesFromPlaylist(playlistContent.toString())
  85
  86         const fd = await open(videoPath, 'r')
  87         for (const range of ranges) {
  88           const buf = Buffer.alloc(range.length)
  89           await read(fd, buf, 0, range.length, range.offset)
  90
  91           rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
  92         }
  93         await close(fd)
  94
  95         const videoFilename = file.filename
  96         json[videoFilename] = rangeHashes
  97       })
  98     })
  99   }
 100
 101   const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
 102   await outputJSON(outputPath, json)
 103
 104   if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
 105     await storeHLSFile(playlist, playlist.segmentsSha256Filename)
 106     await remove(outputPath)
 107   }
 108 }
 109
 110 async function buildSha256Segment (segmentPath: string) {
 111   const buf = await readFile(segmentPath)
 112   return sha256(buf)
 113 }
 114
 115 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
 116   let timer
 117   let remainingBodyKBLimit = bodyKBLimit
 118
 119   logger.info('Importing HLS playlist %s', playlistUrl)
 120
 121   return new Promise<void>(async (res, rej) => {
 122     const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
 123
 124     await ensureDir(tmpDirectory)
 125
 126     timer = setTimeout(() => {
 127       deleteTmpDirectory(tmpDirectory)
 128
 129       return rej(new Error('HLS download timeout.'))
 130     }, timeout)
 131
 132     try {
 133       // Fetch master playlist
 134       const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
 135
 136       const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
 137       const fileUrls = uniq(flatten(await Promise.all(subRequests)))
 138
 139       logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
 140
 141       for (const fileUrl of fileUrls) {
 142         const destPath = join(tmpDirectory, basename(fileUrl))
 143
 144         await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
 145
 146         const { size } = await stat(destPath)
 147         remainingBodyKBLimit -= (size / 1000)
 148
 149         logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
 150       }
 151
 152       clearTimeout(timer)
 153
 154       await move(tmpDirectory, destinationDir, { overwrite: true })
 155
 156       return res()
 157     } catch (err) {
 158       deleteTmpDirectory(tmpDirectory)
 159
 160       return rej(err)
 161     }
 162   })
 163
 164   function deleteTmpDirectory (directory: string) {
 165     remove(directory)
 166       .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
 167   }
 168
 169   async function fetchUniqUrls (playlistUrl: string) {
 170     const { body } = await doRequest(playlistUrl)
 171
 172     if (!body) return []
 173
 174     const urls = body.split('\n')
 175       .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
 176       .map(url => {
 177         if (url.startsWith('http://') || url.startsWith('https://')) return url
 178
 179         return `${dirname(playlistUrl)}/${url}`
 180       })
 181
 182     return uniq(urls)
 183   }
 184 }
 185
 186 // ---------------------------------------------------------------------------
 187
 188 export {
 189   updateMasterHLSPlaylist,
 190   updateSha256VODSegments,
 191   buildSha256Segment,
 192   downloadPlaylistSegments,
 193   updateStreamingPlaylistsInfohashesIfNeeded
 194 }
 195
 196 // ---------------------------------------------------------------------------
 197
 198 function getRangesFromPlaylist (playlistContent: string) {
 199   const ranges: { offset: number, length: number }[] = []
 200   const lines = playlistContent.split('\n')
 201   const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
 202
 203   for (const line of lines) {
 204     const captured = regex.exec(line)
 205
 206     if (captured) {
 207       ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
 208     }
 209   }
 210
 211   return ranges
 212 }