1 import { basename, dirname, join } from 'path'
2 import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
3 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
4 import { getVideoStreamSize, getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg-utils'
5 import { sha256 } from '../helpers/core-utils'
6 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
7 import { logger } from '../helpers/logger'
8 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
9 import { generateRandomString } from '../helpers/utils'
10 import { flatten, uniq } from 'lodash'
11 import { VideoFileModel } from '../models/video/video-file'
12 import { CONFIG } from '../initializers/config'
13 import { sequelizeTypescript } from '../initializers/database'
14 import { MVideoWithFile } from '@server/typings/models'
15 import { getVideoFilename, getVideoFilePath } from './video-paths'
17 async function updateStreamingPlaylistsInfohashesIfNeeded () {
18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
20 // Use separate SQL queries, because we could have many videos to update
21 for (const playlist of playlistsToUpdate) {
22 await sequelizeTypescript.transaction(async t => {
23 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
25 playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
26 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
27 await playlist.save({ transaction: t })
32 async function updateMasterHLSPlaylist (video: MVideoWithFile) {
33 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
34 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
35 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
36 const streamingPlaylist = video.getHLSPlaylist()
38 for (const file of streamingPlaylist.VideoFiles) {
39 // If we did not generated a playlist for this resolution, skip
40 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
41 if (await pathExists(filePlaylistPath) === false) continue
43 const videoFilePath = getVideoFilePath(streamingPlaylist, file)
45 const size = await getVideoStreamSize(videoFilePath)
47 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
48 const resolution = `RESOLUTION=${size.width}x${size.height}`
50 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
51 if (file.fps) line += ',FRAME-RATE=' + file.fps
53 const audioCodec = await getAudioStreamCodec(videoFilePath)
54 const videoCodec = await getVideoStreamCodec(videoFilePath)
55 line += `,CODECS="${videoCodec},${audioCodec}"`
57 masterPlaylists.push(line)
58 masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
61 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
64 async function updateSha256Segments (video: MVideoWithFile) {
65 const json: { [filename: string]: { [range: string]: string } } = {}
67 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
68 const hlsPlaylist = video.getHLSPlaylist()
70 // For all the resolutions available for this video
71 for (const file of hlsPlaylist.VideoFiles) {
72 const rangeHashes: { [range: string]: string } = {}
74 const videoPath = getVideoFilePath(hlsPlaylist, file)
75 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
77 // Maybe the playlist is not generated for this resolution yet
78 if (!await pathExists(playlistPath)) continue
80 const playlistContent = await readFile(playlistPath)
81 const ranges = getRangesFromPlaylist(playlistContent.toString())
83 const fd = await open(videoPath, 'r')
84 for (const range of ranges) {
85 const buf = Buffer.alloc(range.length)
86 await read(fd, buf, 0, range.length, range.offset)
88 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
92 const videoFilename = getVideoFilename(hlsPlaylist, file)
93 json[videoFilename] = rangeHashes
96 const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
97 await outputJSON(outputPath, json)
100 function getRangesFromPlaylist (playlistContent: string) {
101 const ranges: { offset: number, length: number }[] = []
102 const lines = playlistContent.split('\n')
103 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
105 for (const line of lines) {
106 const captured = regex.exec(line)
109 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })
116 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
119 logger.info('Importing HLS playlist %s', playlistUrl)
121 return new Promise<string>(async (res, rej) => {
122 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
124 await ensureDir(tmpDirectory)
126 timer = setTimeout(() => {
127 deleteTmpDirectory(tmpDirectory)
129 return rej(new Error('HLS download timeout.'))
133 // Fetch master playlist
134 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
136 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
137 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
139 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
141 for (const fileUrl of fileUrls) {
142 const destPath = join(tmpDirectory, basename(fileUrl))
144 const bodyKBLimit = 10 * 1000 * 1000 // 10GB
145 await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
150 await move(tmpDirectory, destinationDir, { overwrite: true })
154 deleteTmpDirectory(tmpDirectory)
160 function deleteTmpDirectory (directory: string) {
162 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
165 async function fetchUniqUrls (playlistUrl: string) {
166 const { body } = await doRequest<string>({ uri: playlistUrl })
170 const urls = body.split('\n')
171 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
173 if (url.startsWith('http://') || url.startsWith('https://')) return url
175 return `${dirname(playlistUrl)}/${url}`
182 // ---------------------------------------------------------------------------
185 updateMasterHLSPlaylist,
186 updateSha256Segments,
187 downloadPlaylistSegments,
188 updateStreamingPlaylistsInfohashesIfNeeded
191 // ---------------------------------------------------------------------------