import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
-import { flatten, uniq } from 'lodash'
+import { flatten } from 'lodash'
+import PQueue from 'p-queue'
import { basename, dirname, join } from 'path'
-import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
-import { sha256 } from '../helpers/core-utils'
-import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
+import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
+import { uniqify } from '@shared/core-utils'
+import { sha256 } from '@shared/extra-utils'
+import { VideoStorage } from '@shared/models'
+import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamDimensionsInfo } from '../helpers/ffmpeg'
import { logger } from '../helpers/logger'
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
import { generateRandomString } from '../helpers/utils'
import { sequelizeTypescript } from '../initializers/database'
import { VideoFileModel } from '../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
-import { getHlsResolutionPlaylistFilename } from './paths'
+import { storeHLSFileFromFilename } from './object-storage'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths'
import { VideoPathManager } from './video-path-manager'
async function updateStreamingPlaylistsInfohashesIfNeeded () {
}
}
-async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) {
- const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
+async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) {
+ try {
+ let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist)
+ playlistWithFiles = await updateSha256VODSegments(video, playlist)
- for (const file of playlist.VideoFiles) {
- const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
+ // Refresh playlist, operations can take some time
+ playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)
+ playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
+ await playlistWithFiles.save()
- await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
- const size = await getVideoStreamSize(videoFilePath)
+ video.setHLSPlaylist(playlistWithFiles)
+ } catch (err) {
+ logger.info('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err })
+ }
+}
- const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
- const resolution = `RESOLUTION=${size.width}x${size.height}`
+// ---------------------------------------------------------------------------
- let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
- if (file.fps) line += ',FRAME-RATE=' + file.fps
+// Avoid concurrency issues when updating streaming playlist files
+const playlistFilesQueue = new PQueue({ concurrency: 1 })
- const codecs = await Promise.all([
- getVideoStreamCodec(videoFilePath),
- getAudioStreamCodec(videoFilePath)
- ])
+function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
+ return playlistFilesQueue.add(async () => {
+ const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
- line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
+ const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
- masterPlaylists.push(line)
- masterPlaylists.push(playlistFilename)
- })
- }
+ for (const file of playlist.VideoFiles) {
+ const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
+
+ await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
+ const size = await getVideoStreamDimensionsInfo(videoFilePath)
+
+ const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
+ const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}`
+
+ let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
+ if (file.fps) line += ',FRAME-RATE=' + file.fps
+
+ const codecs = await Promise.all([
+ getVideoStreamCodec(videoFilePath),
+ getAudioStreamCodec(videoFilePath)
+ ])
+
+ line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
+
+ masterPlaylists.push(line)
+ masterPlaylists.push(playlistFilename)
+ })
+ }
+
+ if (playlist.playlistFilename) {
+ await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename)
+ }
+ playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
+
+ const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
+ await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+
+ if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+ playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
+ await remove(masterPlaylistPath)
+ }
- await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
- return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+ return playlist.save()
})
}
-async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) {
- const json: { [filename: string]: { [range: string]: string } } = {}
+// ---------------------------------------------------------------------------
+
+function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise<MStreamingPlaylistFilesVideo> {
+ return playlistFilesQueue.add(async () => {
+ const json: { [filename: string]: { [range: string]: string } } = {}
+
+ const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id)
- // For all the resolutions available for this video
- for (const file of playlist.VideoFiles) {
- const rangeHashes: { [range: string]: string } = {}
- const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
+ // For all the resolutions available for this video
+ for (const file of playlist.VideoFiles) {
+ const rangeHashes: { [range: string]: string } = {}
+ const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
- await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
+ await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
- return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
- const playlistContent = await readFile(resolutionPlaylistPath)
- const ranges = getRangesFromPlaylist(playlistContent.toString())
+ return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
+ const playlistContent = await readFile(resolutionPlaylistPath)
+ const ranges = getRangesFromPlaylist(playlistContent.toString())
- const fd = await open(videoPath, 'r')
- for (const range of ranges) {
- const buf = Buffer.alloc(range.length)
- await read(fd, buf, 0, range.length, range.offset)
+ const fd = await open(videoPath, 'r')
+ for (const range of ranges) {
+ const buf = Buffer.alloc(range.length)
+ await read(fd, buf, 0, range.length, range.offset)
- rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
- }
- await close(fd)
+ rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
+ }
+ await close(fd)
- const videoFilename = file.filename
- json[videoFilename] = rangeHashes
+ const videoFilename = file.filename
+ json[videoFilename] = rangeHashes
+ })
})
- })
- }
+ }
+
+ if (playlist.segmentsSha256Filename) {
+ await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename)
+ }
+ playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
+
+ const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
+ await outputJSON(outputPath, json)
- const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
- await outputJSON(outputPath, json)
+ if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
+ playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
+ await remove(outputPath)
+ }
+
+ return playlist.save()
+ })
}
+// ---------------------------------------------------------------------------
+
async function buildSha256Segment (segmentPath: string) {
const buf = await readFile(segmentPath)
return sha256(buf)
const subPlaylistUrls = await fetchUniqUrls(playlistUrl)
const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u))
- const fileUrls = uniq(flatten(await Promise.all(subRequests)))
+ const fileUrls = uniqify(flatten(await Promise.all(subRequests)))
logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls })
for (const fileUrl of fileUrls) {
const destPath = join(tmpDirectory, basename(fileUrl))
- await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.FILE })
+ await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
const { size } = await stat(destPath)
remainingBodyKBLimit -= (size / 1000)
return `${dirname(playlistUrl)}/${url}`
})
- return uniq(urls)
+ return uniqify(urls)
}
}
// ---------------------------------------------------------------------------
+function injectQueryToPlaylistUrls (content: string, queryString: string) {
+ return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString)
+}
+
+// ---------------------------------------------------------------------------
+
export {
updateMasterHLSPlaylist,
updateSha256VODSegments,
buildSha256Segment,
downloadPlaylistSegments,
- updateStreamingPlaylistsInfohashesIfNeeded
+ updateStreamingPlaylistsInfohashesIfNeeded,
+ updatePlaylistAfterFileChange,
+ injectQueryToPlaylistUrls
}
// ---------------------------------------------------------------------------
projects / github / Chocobozzz / PeerTube.git / blobdiff