server/lib/hls.ts

   1 import { basename, dirname, join } from 'path'
   2 import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
   3 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
   4 import { getVideoStreamSize, getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg-utils'
   5 import { sha256 } from '../helpers/core-utils'
   6 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
   7 import { logger } from '../helpers/logger'
   8 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
   9 import { generateRandomString } from '../helpers/utils'
  10 import { flatten, uniq } from 'lodash'
  11 import { VideoFileModel } from '../models/video/video-file'
  12 import { CONFIG } from '../initializers/config'
  13 import { sequelizeTypescript } from '../initializers/database'
  14 import { MVideoWithFile } from '@server/types/models'
  15 import { getVideoFilename, getVideoFilePath } from './video-paths'
  16
  17 async function updateStreamingPlaylistsInfohashesIfNeeded () {
  18   const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
  19
  20   // Use separate SQL queries, because we could have many videos to update
  21   for (const playlist of playlistsToUpdate) {
  22     await sequelizeTypescript.transaction(async t => {
  23       const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
  24
  25       playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
  26       playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
  27       await playlist.save({ transaction: t })
  28     })
  29   }
  30 }
  31
  32 async function updateMasterHLSPlaylist (video: MVideoWithFile) {
  33   const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
  34   const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
  35   const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
  36   const streamingPlaylist = video.getHLSPlaylist()
  37
  38   for (const file of streamingPlaylist.VideoFiles) {
  39     // If we did not generated a playlist for this resolution, skip
  40     const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  41     if (await pathExists(filePlaylistPath) === false) continue
  42
  43     const videoFilePath = getVideoFilePath(streamingPlaylist, file)
  44
  45     const size = await getVideoStreamSize(videoFilePath)
  46
  47     const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
  48     const resolution = `RESOLUTION=${size.width}x${size.height}`
  49
  50     let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
  51     if (file.fps) line += ',FRAME-RATE=' + file.fps
  52
  53     const videoCodec = await getVideoStreamCodec(videoFilePath)
  54     line += `,CODECS="${videoCodec}`
  55
  56     const audioCodec = await getAudioStreamCodec(videoFilePath)
  57     if (audioCodec) line += `,${audioCodec}`
  58
  59     line += '"'
  60
  61     masterPlaylists.push(line)
  62     masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  63   }
  64
  65   await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
  66 }
  67
  68 async function updateSha256VODSegments (video: MVideoWithFile) {
  69   const json: { [filename: string]: { [range: string]: string } } = {}
  70
  71   const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
  72   const hlsPlaylist = video.getHLSPlaylist()
  73
  74   // For all the resolutions available for this video
  75   for (const file of hlsPlaylist.VideoFiles) {
  76     const rangeHashes: { [range: string]: string } = {}
  77
  78     const videoPath = getVideoFilePath(hlsPlaylist, file)
  79     const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
  80
  81     // Maybe the playlist is not generated for this resolution yet
  82     if (!await pathExists(playlistPath)) continue
  83
  84     const playlistContent = await readFile(playlistPath)
  85     const ranges = getRangesFromPlaylist(playlistContent.toString())
  86
  87     const fd = await open(videoPath, 'r')
  88     for (const range of ranges) {
  89       const buf = Buffer.alloc(range.length)
  90       await read(fd, buf, 0, range.length, range.offset)
  91
  92       rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
  93     }
  94     await close(fd)
  95
  96     const videoFilename = getVideoFilename(hlsPlaylist, file)
  97     json[videoFilename] = rangeHashes
  98   }
  99
 100   const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
 101   await outputJSON(outputPath, json)
 102 }
 103
 104 async function buildSha256Segment (segmentPath: string) {
 105   const buf = await readFile(segmentPath)
 106   return sha256(buf)
 107 }
 108
 109 function getRangesFromPlaylist (playlistContent: string) {
 110   const ranges: { offset: number, length: number }[] = []
 111   const lines = playlistContent.split('\n')
 112   const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
 113
 114   for (const line of lines) {
 115     const captured = regex.exec(line)
 116
 117     if (captured) {
 118       ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
 119     }
 120   }
 121
 122   return ranges
 123 }
 124
 125 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
 126   let timer
 127
 128   logger.info('Importing HLS playlist %s', playlistUrl)
 129
 130   return new Promise<string>(async (res, rej) => {
 131     const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
 132
 133     await ensureDir(tmpDirectory)
 134
 135     timer = setTimeout(() => {
 136       deleteTmpDirectory(tmpDirectory)
 137
 138       return rej(new Error('HLS download timeout.'))
 139     }, timeout)
 140
 141     try {
 142       // Fetch master playlist
 143       const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
 144
 145       const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
 146       const fileUrls = uniq(flatten(await Promise.all(subRequests)))
 147
 148       logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
 149
 150       for (const fileUrl of fileUrls) {
 151         const destPath = join(tmpDirectory, basename(fileUrl))
 152
 153         const bodyKBLimit = 10 * 1000 * 1000 // 10GB
 154         await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
 155       }
 156
 157       clearTimeout(timer)
 158
 159       await move(tmpDirectory, destinationDir, { overwrite: true })
 160
 161       return res()
 162     } catch (err) {
 163       deleteTmpDirectory(tmpDirectory)
 164
 165       return rej(err)
 166     }
 167   })
 168
 169   function deleteTmpDirectory (directory: string) {
 170     remove(directory)
 171       .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
 172   }
 173
 174   async function fetchUniqUrls (playlistUrl: string) {
 175     const { body } = await doRequest<string>({ uri: playlistUrl })
 176
 177     if (!body) return []
 178
 179     const urls = body.split('\n')
 180       .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
 181       .map(url => {
 182         if (url.startsWith('http://') || url.startsWith('https://')) return url
 183
 184         return `${dirname(playlistUrl)}/${url}`
 185       })
 186
 187     return uniq(urls)
 188   }
 189 }
 190
 191 // ---------------------------------------------------------------------------
 192
 193 export {
 194   updateMasterHLSPlaylist,
 195   updateSha256VODSegments,
 196   buildSha256Segment,
 197   downloadPlaylistSegments,
 198   updateStreamingPlaylistsInfohashesIfNeeded
 199 }
 200
 201 // ---------------------------------------------------------------------------