1 import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2 import { flatten, uniq } from 'lodash'
3 import PQueue from 'p-queue'
4 import { basename, dirname, join } from 'path'
5 import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
6 import { sha256 } from '@shared/extra-utils'
7 import { VideoStorage } from '@shared/models'
8 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
9 import { logger } from '../helpers/logger'
10 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
11 import { generateRandomString } from '../helpers/utils'
12 import { CONFIG } from '../initializers/config'
13 import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants'
14 import { sequelizeTypescript } from '../initializers/database'
15 import { VideoFileModel } from '../models/video/video-file'
16 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
17 import { storeHLSFile } from './object-storage'
18 import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
19 import { VideoPathManager } from './video-path-manager'
21 async function updateStreamingPlaylistsInfohashesIfNeeded () {
22 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
24 // Use separate SQL queries, because we could have many videos to update
25 for (const playlist of playlistsToUpdate) {
26 await sequelizeTypescript.transaction(async t => {
27 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
29 playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
30 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
32 await playlist.save({ transaction: t })
37 async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) {
39 let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist)
40 playlistWithFiles = await updateSha256VODSegments(video, playlist)
42 // Refresh playlist, operations can take some time
43 playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)
44 playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
45 await playlistWithFiles.save()
47 video.setHLSPlaylist(playlistWithFiles)
49 logger.info('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err })
53 // ---------------------------------------------------------------------------
55 // Avoid concurrency issues when updating streaming playlist files
56 const playlistFilesQueue = new PQueue({ concurrency: 1 })
58 function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
59 return playlistFilesQueue.add(async () => {
60 const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
62 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
64 for (const file of playlist.VideoFiles) {
65 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
67 await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
68 const size = await getVideoStreamDimensionsInfo(videoFilePath)
70 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
71 const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
73 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
74 if (file.fps) line += ',FRAME-RATE=' + file.fps
76 const codecs = await Promise.all([
77 getVideoStreamCodec(videoFilePath),
78 getAudioStreamCodec(videoFilePath)
81 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
83 masterPlaylists.push(line)
84 masterPlaylists.push(playlistFilename)
88 if (playlist.playlistFilename) {
89 await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename)
91 playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
93 const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
94 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
96 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
97 playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename)
98 await remove(masterPlaylistPath)
101 return playlist.save()
105 // ---------------------------------------------------------------------------
107 function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
108 return playlistFilesQueue.add(async () => {
109 const json: { [filename: string]: { [range: string]: string } } = {}
111 const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
113 // For all the resolutions available for this video
114 for (const file of playlist.VideoFiles) {
115 const rangeHashes: { [range: string]: string } = {}
116 const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
118 await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
120 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
121 const playlistContent = await readFile(resolutionPlaylistPath)
122 const ranges = getRangesFromPlaylist(playlistContent.toString())
124 const fd = await open(videoPath, 'r')
125 for (const range of ranges) {
126 const buf = Buffer.alloc(range.length)
127 await read(fd, buf, 0, range.length, range.offset)
129 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
133 const videoFilename = file.filename
134 json[videoFilename] = rangeHashes
139 if (playlist.segmentsSha256Filename) {
140 await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename)
142 playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
144 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
145 await outputJSON(outputPath, json)
147 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
148 playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename)
149 await remove(outputPath)
152 return playlist.save()
156 // ---------------------------------------------------------------------------
158 async function buildSha256Segment (segmentPath: string) {
159 const buf = await readFile(segmentPath)
163 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
165 let remainingBodyKBLimit = bodyKBLimit
167 logger.info('Importing HLS playlist %s', playlistUrl)
169 return new Promise<void>(async (res, rej) => {
170 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
172 await ensureDir(tmpDirectory)
174 timer = setTimeout(() => {
175 deleteTmpDirectory(tmpDirectory)
177 return rej(new Error('HLS download timeout.'))
181 // Fetch master playlist
182 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
184 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
185 const fileUrls = uniq(flatten(await Promise.all(subRequests)))
187 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
189 for (const fileUrl of fileUrls) {
190 const destPath = join(tmpDirectory, basename(fileUrl))
192 await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
194 const { size } = await stat(destPath)
195 remainingBodyKBLimit -= (size / 1000)
197 logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
202 await move(tmpDirectory, destinationDir, { overwrite: true })
206 deleteTmpDirectory(tmpDirectory)
212 function deleteTmpDirectory (directory: string) {
214 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
217 async function fetchUniqUrls (playlistUrl: string) {
218 const { body } = await doRequest(playlistUrl)
222 const urls = body.split('\n')
223 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
225 if (url.startsWith('http://') || url.startsWith('https://')) return url
227 return `${dirname(playlistUrl)}/${url}`
234 // ---------------------------------------------------------------------------
237 updateMasterHLSPlaylist,
238 updateSha256VODSegments,
240 downloadPlaylistSegments,
241 updateStreamingPlaylistsInfohashesIfNeeded,
242 updatePlaylistAfterFileChange
245 // ---------------------------------------------------------------------------
247 function getRangesFromPlaylist (playlistContent: string) {
248 const ranges: { offset: number, length: number }[] = []
249 const lines = playlistContent.split('\n')
250 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
252 for (const line of lines) {
253 const captured = regex.exec(line)
256 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })