server/lib/hls.ts

   1 import { VideoModel } from '../models/video/video'
   2 import { basename, dirname, join } from 'path'
   3 import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, sequelizeTypescript } from '../initializers'
   4 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
   5 import { getVideoFileSize } from '../helpers/ffmpeg-utils'
   6 import { sha256 } from '../helpers/core-utils'
   7 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
   8 import { logger } from '../helpers/logger'
   9 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
  10 import { generateRandomString } from '../helpers/utils'
  11 import { flatten, uniq } from 'lodash'
  12 import { VideoFileModel } from '../models/video/video-file'
  13
  14 async function updateStreamingPlaylistsInfohashesIfNeeded () {
  15   const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
  16
  17   // Use separate SQL queries, because we could have many videos to update
  18   for (const playlist of playlistsToUpdate) {
  19     await sequelizeTypescript.transaction(async t => {
  20       const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
  21
  22       playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
  23       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
  24       await playlist.save({ transaction: t })
  25     })
  26   }
  27 }
  28
  29 async function updateMasterHLSPlaylist (video: VideoModel) {
  30   const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
  31   const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
  32   const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
  33
  34   for (const file of video.VideoFiles) {
  35     // If we did not generated a playlist for this resolution, skip
  36     const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  37     if (await pathExists(filePlaylistPath) === false) continue
  38
  39     const videoFilePath = video.getVideoFilePath(file)
  40
  41     const size = await getVideoFileSize(videoFilePath)
  42
  43     const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
  44     const resolution = `RESOLUTION=${size.width}x${size.height}`
  45
  46     let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
  47     if (file.fps) line += ',FRAME-RATE=' + file.fps
  48
  49     masterPlaylists.push(line)
  50     masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  51   }
  52
  53   await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
  54 }
  55
  56 async function updateSha256Segments (video: VideoModel) {
  57   const json: { [filename: string]: { [range: string]: string } } = {}
  58
  59   const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
  60
  61   // For all the resolutions available for this video
  62   for (const file of video.VideoFiles) {
  63     const rangeHashes: { [range: string]: string } = {}
  64
  65     const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
  66     const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  67
  68     // Maybe the playlist is not generated for this resolution yet
  69     if (!await pathExists(playlistPath)) continue
  70
  71     const playlistContent = await readFile(playlistPath)
  72     const ranges = getRangesFromPlaylist(playlistContent.toString())
  73
  74     const fd = await open(videoPath, 'r')
  75     for (const range of ranges) {
  76       const buf = Buffer.alloc(range.length)
  77       await read(fd, buf, 0, range.length, range.offset)
  78
  79       rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
  80     }
  81     await close(fd)
  82
  83     const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
  84     json[videoFilename] = rangeHashes
  85   }
  86
  87   const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
  88   await outputJSON(outputPath, json)
  89 }
  90
  91 function getRangesFromPlaylist (playlistContent: string) {
  92   const ranges: { offset: number, length: number }[] = []
  93   const lines = playlistContent.split('\n')
  94   const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
  95
  96   for (const line of lines) {
  97     const captured = regex.exec(line)
  98
  99     if (captured) {
 100       ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
 101     }
 102   }
 103
 104   return ranges
 105 }
 106
 107 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
 108   let timer
 109
 110   logger.info('Importing HLS playlist %s', playlistUrl)
 111
 112   return new Promise<string>(async (res, rej) => {
 113     const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
 114
 115     await ensureDir(tmpDirectory)
 116
 117     timer = setTimeout(() => {
 118       deleteTmpDirectory(tmpDirectory)
 119
 120       return rej(new Error('HLS download timeout.'))
 121     }, timeout)
 122
 123     try {
 124       // Fetch master playlist
 125       const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
 126
 127       const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
 128       const fileUrls = uniq(flatten(await Promise.all(subRequests)))
 129
 130       logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
 131
 132       for (const fileUrl of fileUrls) {
 133         const destPath = join(tmpDirectory, basename(fileUrl))
 134
 135         const bodyKBLimit = 10 * 1000 * 1000 // 10GB
 136         await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
 137       }
 138
 139       clearTimeout(timer)
 140
 141       await move(tmpDirectory, destinationDir, { overwrite: true })
 142
 143       return res()
 144     } catch (err) {
 145       deleteTmpDirectory(tmpDirectory)
 146
 147       return rej(err)
 148     }
 149   })
 150
 151   function deleteTmpDirectory (directory: string) {
 152     remove(directory)
 153       .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
 154   }
 155
 156   async function fetchUniqUrls (playlistUrl: string) {
 157     const { body } = await doRequest<string>({ uri: playlistUrl })
 158
 159     if (!body) return []
 160
 161     const urls = body.split('\n')
 162       .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
 163       .map(url => {
 164         if (url.startsWith('http://') || url.startsWith('https://')) return url
 165
 166         return `${dirname(playlistUrl)}/${url}`
 167       })
 168
 169     return uniq(urls)
 170   }
 171 }
 172
 173 // ---------------------------------------------------------------------------
 174
 175 export {
 176   updateMasterHLSPlaylist,
 177   updateSha256Segments,
 178   downloadPlaylistSegments,
 179   updateStreamingPlaylistsInfohashesIfNeeded
 180 }
 181
 182 // ---------------------------------------------------------------------------