server/lib/hls.ts

   1 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
   2 import { flatten, uniq } from 'lodash'
   3 import { basename, dirname, join } from 'path'
   4 import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
   5 import { sha256 } from '../helpers/core-utils'
   6 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
   7 import { logger } from '../helpers/logger'
   8 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
   9 import { generateRandomString } from '../helpers/utils'
  10 import { CONFIG } from '../initializers/config'
  11 import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
  12 import { sequelizeTypescript } from '../initializers/database'
  13 import { VideoFileModel } from '../models/video/video-file'
  14 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
  15 import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths'
  16
  17 async function updateStreamingPlaylistsInfohashesIfNeeded () {
  18   const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
  19
  20   // Use separate SQL queries, because we could have many videos to update
  21   for (const playlist of playlistsToUpdate) {
  22     await sequelizeTypescript.transaction(async t => {
  23       const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
  24
  25       playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
  26       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
  27
  28       await playlist.save({ transaction: t })
  29     })
  30   }
  31 }
  32
  33 async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
  34   const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
  35
  36   const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
  37
  38   const masterPlaylistPath = join(directory, playlist.playlistFilename)
  39
  40   for (const file of playlist.VideoFiles) {
  41     const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
  42
  43     // If we did not generated a playlist for this resolution, skip
  44     const filePlaylistPath = join(directory, playlistFilename)
  45     if (await pathExists(filePlaylistPath) === false) continue
  46
  47     const videoFilePath = getVideoFilePath(playlist, file)
  48
  49     const size = await getVideoStreamSize(videoFilePath)
  50
  51     const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
  52     const resolution = `RESOLUTION=${size.width}x${size.height}`
  53
  54     let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
  55     if (file.fps) line += ',FRAME-RATE=' + file.fps
  56
  57     const codecs = await Promise.all([
  58       getVideoStreamCodec(videoFilePath),
  59       getAudioStreamCodec(videoFilePath)
  60     ])
  61
  62     line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
  63
  64     masterPlaylists.push(line)
  65     masterPlaylists.push(playlistFilename)
  66   }
  67
  68   await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
  69 }
  70
  71 async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
  72   const json: { [filename: string]: { [range: string]: string } } = {}
  73
  74   const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
  75
  76   // For all the resolutions available for this video
  77   for (const file of playlist.VideoFiles) {
  78     const rangeHashes: { [range: string]: string } = {}
  79
  80     const videoPath = getVideoFilePath(playlist, file)
  81     const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
  82
  83     // Maybe the playlist is not generated for this resolution yet
  84     if (!await pathExists(resolutionPlaylistPath)) continue
  85
  86     const playlistContent = await readFile(resolutionPlaylistPath)
  87     const ranges = getRangesFromPlaylist(playlistContent.toString())
  88
  89     const fd = await open(videoPath, 'r')
  90     for (const range of ranges) {
  91       const buf = Buffer.alloc(range.length)
  92       await read(fd, buf, 0, range.length, range.offset)
  93
  94       rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
  95     }
  96     await close(fd)
  97
  98     const videoFilename = file.filename
  99     json[videoFilename] = rangeHashes
 100   }
 101
 102   const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename)
 103   await outputJSON(outputPath, json)
 104 }
 105
 106 async function buildSha256Segment (segmentPath: string) {
 107   const buf = await readFile(segmentPath)
 108   return sha256(buf)
 109 }
 110
 111 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
 112   let timer
 113
 114   logger.info('Importing HLS playlist %s', playlistUrl)
 115
 116   return new Promise<void>(async (res, rej) => {
 117     const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
 118
 119     await ensureDir(tmpDirectory)
 120
 121     timer = setTimeout(() => {
 122       deleteTmpDirectory(tmpDirectory)
 123
 124       return rej(new Error('HLS download timeout.'))
 125     }, timeout)
 126
 127     try {
 128       // Fetch master playlist
 129       const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
 130
 131       const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
 132       const fileUrls = uniq(flatten(await Promise.all(subRequests)))
 133
 134       logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
 135
 136       for (const fileUrl of fileUrls) {
 137         const destPath = join(tmpDirectory, basename(fileUrl))
 138
 139         const bodyKBLimit = 10 * 1000 * 1000 // 10GB
 140         await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
 141       }
 142
 143       clearTimeout(timer)
 144
 145       await move(tmpDirectory, destinationDir, { overwrite: true })
 146
 147       return res()
 148     } catch (err) {
 149       deleteTmpDirectory(tmpDirectory)
 150
 151       return rej(err)
 152     }
 153   })
 154
 155   function deleteTmpDirectory (directory: string) {
 156     remove(directory)
 157       .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
 158   }
 159
 160   async function fetchUniqUrls (playlistUrl: string) {
 161     const { body } = await doRequest(playlistUrl)
 162
 163     if (!body) return []
 164
 165     const urls = body.split('\n')
 166       .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
 167       .map(url => {
 168         if (url.startsWith('http://') || url.startsWith('https://')) return url
 169
 170         return `${dirname(playlistUrl)}/${url}`
 171       })
 172
 173     return uniq(urls)
 174   }
 175 }
 176
 177 // ---------------------------------------------------------------------------
 178
 179 export {
 180   updateMasterHLSPlaylist,
 181   updateSha256VODSegments,
 182   buildSha256Segment,
 183   downloadPlaylistSegments,
 184   updateStreamingPlaylistsInfohashesIfNeeded
 185 }
 186
 187 // ---------------------------------------------------------------------------
 188
 189 function getRangesFromPlaylist (playlistContent: string) {
 190   const ranges: { offset: number, length: number }[] = []
 191   const lines = playlistContent.split('\n')
 192   const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
 193
 194   for (const line of lines) {
 195     const captured = regex.exec(line)
 196
 197     if (captured) {
 198       ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
 199     }
 200   }
 201
 202   return ranges
 203 }