server/lib/hls.ts

   1 import { VideoModel } from '../models/video/video'
   2 import { basename, join, dirname } from 'path'
   3 import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
   4 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
   5 import { getVideoFileSize } from '../helpers/ffmpeg-utils'
   6 import { sha256 } from '../helpers/core-utils'
   7 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
   8 import { logger } from '../helpers/logger'
   9 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
  10 import { generateRandomString } from '../helpers/utils'
  11 import { flatten, uniq } from 'lodash'
  12
  13 async function updateMasterHLSPlaylist (video: VideoModel) {
  14   const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
  15   const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
  16   const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
  17
  18   for (const file of video.VideoFiles) {
  19     // If we did not generated a playlist for this resolution, skip
  20     const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  21     if (await pathExists(filePlaylistPath) === false) continue
  22
  23     const videoFilePath = video.getVideoFilePath(file)
  24
  25     const size = await getVideoFileSize(videoFilePath)
  26
  27     const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
  28     const resolution = `RESOLUTION=${size.width}x${size.height}`
  29
  30     let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
  31     if (file.fps) line += ',FRAME-RATE=' + file.fps
  32
  33     masterPlaylists.push(line)
  34     masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  35   }
  36
  37   await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
  38 }
  39
  40 async function updateSha256Segments (video: VideoModel) {
  41   const json: { [filename: string]: { [range: string]: string } } = {}
  42
  43   const playlistDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
  44
  45   // For all the resolutions available for this video
  46   for (const file of video.VideoFiles) {
  47     const rangeHashes: { [range: string]: string } = {}
  48
  49     const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution))
  50     const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  51
  52     // Maybe the playlist is not generated for this resolution yet
  53     if (!await pathExists(playlistPath)) continue
  54
  55     const playlistContent = await readFile(playlistPath)
  56     const ranges = getRangesFromPlaylist(playlistContent.toString())
  57
  58     const fd = await open(videoPath, 'r')
  59     for (const range of ranges) {
  60       const buf = Buffer.alloc(range.length)
  61       await read(fd, buf, 0, range.length, range.offset)
  62
  63       rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
  64     }
  65     await close(fd)
  66
  67     const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)
  68     json[videoFilename] = rangeHashes
  69   }
  70
  71   const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
  72   await outputJSON(outputPath, json)
  73 }
  74
  75 function getRangesFromPlaylist (playlistContent: string) {
  76   const ranges: { offset: number, length: number }[] = []
  77   const lines = playlistContent.split('\n')
  78   const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
  79
  80   for (const line of lines) {
  81     const captured = regex.exec(line)
  82
  83     if (captured) {
  84       ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
  85     }
  86   }
  87
  88   return ranges
  89 }
  90
  91 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
  92   let timer
  93
  94   logger.info('Importing HLS playlist %s', playlistUrl)
  95
  96   return new Promise<string>(async (res, rej) => {
  97     const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
  98
  99     await ensureDir(tmpDirectory)
 100
 101     timer = setTimeout(() => {
 102       deleteTmpDirectory(tmpDirectory)
 103
 104       return rej(new Error('HLS download timeout.'))
 105     }, timeout)
 106
 107     try {
 108       // Fetch master playlist
 109       const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
 110
 111       const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
 112       const fileUrls = uniq(flatten(await Promise.all(subRequests)))
 113
 114       logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
 115
 116       for (const fileUrl of fileUrls) {
 117         const destPath = join(tmpDirectory, basename(fileUrl))
 118
 119         await doRequestAndSaveToFile({ uri: fileUrl }, destPath)
 120       }
 121
 122       clearTimeout(timer)
 123
 124       await move(tmpDirectory, destinationDir, { overwrite: true })
 125
 126       return res()
 127     } catch (err) {
 128       deleteTmpDirectory(tmpDirectory)
 129
 130       return rej(err)
 131     }
 132   })
 133
 134   function deleteTmpDirectory (directory: string) {
 135     remove(directory)
 136       .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
 137   }
 138
 139   async function fetchUniqUrls (playlistUrl: string) {
 140     const { body } = await doRequest<string>({ uri: playlistUrl })
 141
 142     if (!body) return []
 143
 144     const urls = body.split('\n')
 145       .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
 146       .map(url => {
 147         if (url.startsWith('http://') || url.startsWith('https://')) return url
 148
 149         return `${dirname(playlistUrl)}/${url}`
 150       })
 151
 152     return uniq(urls)
 153   }
 154 }
 155
 156 // ---------------------------------------------------------------------------
 157
 158 export {
 159   updateMasterHLSPlaylist,
 160   updateSha256Segments,
 161   downloadPlaylistSegments
 162 }
 163
 164 // ---------------------------------------------------------------------------