1 import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
2 import { flatten } from 'lodash'
3 import PQueue from 'p-queue'
4 import { basename, dirname, join } from 'path'
5 import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
6 import { uniqify } from '@shared/core-utils'
7 import { sha256 } from '@shared/extra-utils'
8 import { VideoStorage } from '@shared/models'
9 import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
10 import { logger } from '../helpers/logger'
11 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
12 import { generateRandomString } from '../helpers/utils'
13 import { CONFIG } from '../initializers/config'
14 import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants'
15 import { sequelizeTypescript } from '../initializers/database'
16 import { VideoFileModel } from '../models/video/video-file'
17 import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
18 import { storeHLSFile } from './object-storage'
19 import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
20 import { VideoPathManager } from './video-path-manager'
22 async function updateStreamingPlaylistsInfohashesIfNeeded () {
23 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
25 // Use separate SQL queries, because we could have many videos to update
26 for (const playlist of playlistsToUpdate) {
27 await sequelizeTypescript.transaction(async t => {
28 const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
30 playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
31 playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
33 await playlist.save({ transaction: t })
38 async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) {
40 let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist)
41 playlistWithFiles = await updateSha256VODSegments(video, playlist)
43 // Refresh playlist, operations can take some time
44 playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)
45 playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
46 await playlistWithFiles.save()
48 video.setHLSPlaylist(playlistWithFiles)
50 logger.info('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err })
54 // ---------------------------------------------------------------------------
56 // Avoid concurrency issues when updating streaming playlist files
57 const playlistFilesQueue = new PQueue({ concurrency: 1 })
59 function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
60 return playlistFilesQueue.add(async () => {
61 const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
63 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
65 for (const file of playlist.VideoFiles) {
66 const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
68 await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
69 const size = await getVideoStreamDimensionsInfo(videoFilePath)
71 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
72 const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
74 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
75 if (file.fps) line += ',FRAME-RATE=' + file.fps
77 const codecs = await Promise.all([
78 getVideoStreamCodec(videoFilePath),
79 getAudioStreamCodec(videoFilePath)
82 line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
84 masterPlaylists.push(line)
85 masterPlaylists.push(playlistFilename)
89 if (playlist.playlistFilename) {
90 await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename)
92 playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
94 const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
95 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
97 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
98 playlist.playlistUrl = await storeHLSFile(playlist, playlist.playlistFilename)
99 await remove(masterPlaylistPath)
102 return playlist.save()
106 // ---------------------------------------------------------------------------
108 function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
109 return playlistFilesQueue.add(async () => {
110 const json: { [filename: string]: { [range: string]: string } } = {}
112 const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
114 // For all the resolutions available for this video
115 for (const file of playlist.VideoFiles) {
116 const rangeHashes: { [range: string]: string } = {}
117 const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
119 await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
121 return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
122 const playlistContent = await readFile(resolutionPlaylistPath)
123 const ranges = getRangesFromPlaylist(playlistContent.toString())
125 const fd = await open(videoPath, 'r')
126 for (const range of ranges) {
127 const buf = Buffer.alloc(range.length)
128 await read(fd, buf, 0, range.length, range.offset)
130 rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
134 const videoFilename = file.filename
135 json[videoFilename] = rangeHashes
140 if (playlist.segmentsSha256Filename) {
141 await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename)
143 playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
145 const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
146 await outputJSON(outputPath, json)
148 if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
149 playlist.segmentsSha256Url = await storeHLSFile(playlist, playlist.segmentsSha256Filename)
150 await remove(outputPath)
153 return playlist.save()
157 // ---------------------------------------------------------------------------
159 async function buildSha256Segment (segmentPath: string) {
160 const buf = await readFile(segmentPath)
164 function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
166 let remainingBodyKBLimit = bodyKBLimit
168 logger.info('Importing HLS playlist %s', playlistUrl)
170 return new Promise<void>(async (res, rej) => {
171 const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10))
173 await ensureDir(tmpDirectory)
175 timer = setTimeout(() => {
176 deleteTmpDirectory(tmpDirectory)
178 return rej(new Error('HLS download timeout.'))
182 // Fetch master playlist
183 const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
185 const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
186 const fileUrls = uniqify(flatten(await Promise.all(subRequests)))
188 logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
190 for (const fileUrl of fileUrls) {
191 const destPath = join(tmpDirectory, basename(fileUrl))
193 await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
195 const { size } = await stat(destPath)
196 remainingBodyKBLimit -= (size / 1000)
198 logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
203 await move(tmpDirectory, destinationDir, { overwrite: true })
207 deleteTmpDirectory(tmpDirectory)
213 function deleteTmpDirectory (directory: string) {
215 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
218 async function fetchUniqUrls (playlistUrl: string) {
219 const { body } = await doRequest(playlistUrl)
223 const urls = body.split('\n')
224 .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4'))
226 if (url.startsWith('http://') || url.startsWith('https://')) return url
228 return `${dirname(playlistUrl)}/${url}`
235 // ---------------------------------------------------------------------------
238 updateMasterHLSPlaylist,
239 updateSha256VODSegments,
241 downloadPlaylistSegments,
242 updateStreamingPlaylistsInfohashesIfNeeded,
243 updatePlaylistAfterFileChange
246 // ---------------------------------------------------------------------------
248 function getRangesFromPlaylist (playlistContent: string) {
249 const ranges: { offset: number, length: number }[] = []
250 const lines = playlistContent.split('\n')
251 const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/
253 for (const line of lines) {
254 const captured = regex.exec(line)
257 ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) })