-import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
+import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra'
import { flatten, uniq } from 'lodash'
import { basename, dirname, join } from 'path'
-import { MVideoWithFile } from '@server/types/models'
-import { sha256 } from '../helpers/core-utils'
+import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models'
+import { sha256 } from '@shared/extra-utils'
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
import { logger } from '../helpers/logger'
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
import { generateRandomString } from '../helpers/utils'
import { CONFIG } from '../initializers/config'
-import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../initializers/constants'
+import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants'
import { sequelizeTypescript } from '../initializers/database'
import { VideoFileModel } from '../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
-import { getVideoFilePath } from './video-paths'
+import { getHlsResolutionPlaylistFilename } from './paths'
+import { VideoPathManager } from './video-path-manager'
async function updateStreamingPlaylistsInfohashesIfNeeded () {
const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
await sequelizeTypescript.transaction(async t => {
const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
- playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
+ playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
await playlist.save({ transaction: t })
})
}
}
-async function updateMasterHLSPlaylist (video: MVideoWithFile) {
- const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
+async function updateMasterHLSPlaylist (video: MVideo, playlist: MStreamingPlaylistFilesVideo) {
const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
- const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
- const streamingPlaylist = video.getHLSPlaylist()
-
- for (const file of streamingPlaylist.VideoFiles) {
- const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)
-
- // If we did not generated a playlist for this resolution, skip
- const filePlaylistPath = join(directory, playlistFilename)
- if (await pathExists(filePlaylistPath) === false) continue
- const videoFilePath = getVideoFilePath(streamingPlaylist, file)
+ for (const file of playlist.VideoFiles) {
+ const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
- const size = await getVideoStreamSize(videoFilePath)
+ await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => {
+ const size = await getVideoStreamSize(videoFilePath)
- const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
- const resolution = `RESOLUTION=${size.width}x${size.height}`
+ const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
+ const resolution = `RESOLUTION=${size.width}x${size.height}`
- let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
- if (file.fps) line += ',FRAME-RATE=' + file.fps
+ let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
+ if (file.fps) line += ',FRAME-RATE=' + file.fps
- const codecs = await Promise.all([
- getVideoStreamCodec(videoFilePath),
- getAudioStreamCodec(videoFilePath)
- ])
+ const codecs = await Promise.all([
+ getVideoStreamCodec(videoFilePath),
+ getAudioStreamCodec(videoFilePath)
+ ])
- line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
+ line += `,CODECS="${codecs.filter(c => !!c).join(',')}"`
- masterPlaylists.push(line)
- masterPlaylists.push(playlistFilename)
+ masterPlaylists.push(line)
+ masterPlaylists.push(playlistFilename)
+ })
}
- await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+ await VideoPathManager.Instance.makeAvailablePlaylistFile(playlist, playlist.playlistFilename, masterPlaylistPath => {
+ return writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
+ })
}
-async function updateSha256VODSegments (video: MVideoWithFile) {
+async function updateSha256VODSegments (video: MVideoUUID, playlist: MStreamingPlaylistFilesVideo) {
const json: { [filename: string]: { [range: string]: string } } = {}
- const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
- const hlsPlaylist = video.getHLSPlaylist()
-
// For all the resolutions available for this video
- for (const file of hlsPlaylist.VideoFiles) {
+ for (const file of playlist.VideoFiles) {
const rangeHashes: { [range: string]: string } = {}
+ const fileWithPlaylist = file.withVideoOrPlaylist(playlist)
- const videoPath = getVideoFilePath(hlsPlaylist, file)
- const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
+ await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => {
- // Maybe the playlist is not generated for this resolution yet
- if (!await pathExists(playlistPath)) continue
+ return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => {
+ const playlistContent = await readFile(resolutionPlaylistPath)
+ const ranges = getRangesFromPlaylist(playlistContent.toString())
- const playlistContent = await readFile(playlistPath)
- const ranges = getRangesFromPlaylist(playlistContent.toString())
+ const fd = await open(videoPath, 'r')
+ for (const range of ranges) {
+ const buf = Buffer.alloc(range.length)
+ await read(fd, buf, 0, range.length, range.offset)
- const fd = await open(videoPath, 'r')
- for (const range of ranges) {
- const buf = Buffer.alloc(range.length)
- await read(fd, buf, 0, range.length, range.offset)
+ rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
+ }
+ await close(fd)
- rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf)
- }
- await close(fd)
-
- const videoFilename = file.filename
- json[videoFilename] = rangeHashes
+ const videoFilename = file.filename
+ json[videoFilename] = rangeHashes
+ })
+ })
}
- const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+ const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json)
}
return sha256(buf)
}
-function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
+function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) {
let timer
+ let remainingBodyKBLimit = bodyKBLimit
logger.info('Importing HLS playlist %s', playlistUrl)
for (const fileUrl of fileUrls) {
const destPath = join(tmpDirectory, basename(fileUrl))
- const bodyKBLimit = 10 * 1000 * 1000 // 10GB
- await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit })
+ await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY })
+
+ const { size } = await stat(destPath)
+ remainingBodyKBLimit -= (size / 1000)
+
+ logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit))
}
clearTimeout(timer)
projects / github / Chocobozzz / PeerTube.git / blobdiff