process.exit(-1)
})
-let currentVideoId = null
-let currentFile = null
+let currentVideoId: string
+let currentFilePath: string
process.on('SIGINT', async function () {
console.log('Cleaning up temp files')
- await remove(`${currentFile}_backup`)
- await remove(`${dirname(currentFile)}/${currentVideoId}-transcoded.mp4`)
+ await remove(`${currentFilePath}_backup`)
+ await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`)
process.exit(0)
})
currentVideoId = video.id
for (const file of video.VideoFiles) {
- currentFile = getVideoFilePath(video, file)
+ currentFilePath = getVideoFilePath(video, file)
const [ videoBitrate, fps, resolution ] = await Promise.all([
- getVideoFileBitrate(currentFile),
- getVideoFileFPS(currentFile),
- getVideoFileResolution(currentFile)
+ getVideoFileBitrate(currentFilePath),
+ getVideoFileFPS(currentFilePath),
+ getVideoFileResolution(currentFilePath)
])
const maxBitrate = getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)
if (isMaxBitrateExceeded) {
console.log(
'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
- basename(currentFile), videoBitrate / 1000, maxBitrate / 1000
+ basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
)
- const backupFile = `${currentFile}_backup`
- await copy(currentFile, backupFile)
+ const backupFile = `${currentFilePath}_backup`
+ await copy(currentFilePath, backupFile)
await optimizeOriginalVideofile(video, file)
+ // Update file path, the video filename changed
+ currentFilePath = getVideoFilePath(video, file)
const originalDuration = await getDurationFromVideoFile(backupFile)
- const newDuration = await getDurationFromVideoFile(currentFile)
+ const newDuration = await getDurationFromVideoFile(currentFilePath)
if (originalDuration === newDuration) {
- console.log('Finished optimizing %s', basename(currentFile))
+ console.log('Finished optimizing %s', basename(currentFilePath))
await remove(backupFile)
continue
}
- console.log('Failed to optimize %s, restoring original', basename(currentFile))
- await move(backupFile, currentFile, { overwrite: true })
+ console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
+ await move(backupFile, currentFilePath, { overwrite: true })
await createTorrentAndSetInfoHash(video, file)
await file.save()
}
registerTSPaths()
import * as prompt from 'prompt'
-import { join } from 'path'
+import { join, basename } from 'path'
import { CONFIG } from '../server/initializers/config'
import { VideoModel } from '../server/models/video/video'
import { initDatabaseModels } from '../server/initializers/database'
-import { readdir, remove } from 'fs-extra'
+import { readdir, remove, stat } from 'fs-extra'
import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
import * as Bluebird from 'bluebird'
import { getUUIDFromFilename } from '../server/helpers/utils'
import { ActorImageModel } from '../server/models/actor/actor-image'
import { uniq, values } from 'lodash'
import { ThumbnailType } from '@shared/models'
+import { VideoFileModel } from '@server/models/video/video-file'
run()
.then(() => process.exit(0))
console.log('Detecting files to remove, it could take a while...')
toDelete = toDelete.concat(
- await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
- await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
+ await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
+ await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
const toDelete: string[] = []
await Bluebird.map(files, async file => {
- if (await existFun(file) !== true) {
- toDelete.push(join(directory, file))
+ const filePath = join(directory, file)
+
+ if (await existFun(filePath) !== true) {
+ toDelete.push(filePath)
}
}, { concurrency: 20 })
return toDelete
}
-function doesVideoExist (keepOnlyOwned: boolean) {
- return async (file: string) => {
- const uuid = getUUIDFromFilename(file)
- const video = await VideoModel.load(uuid)
+function doesWebTorrentFileExist () {
+ return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
+}
- return video && (keepOnlyOwned === false || video.isOwned())
- }
+function doesTorrentFileExist () {
+ return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
}
function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
- return async (file: string) => {
- const thumbnail = await ThumbnailModel.loadByFilename(file, type)
+ return async (filePath: string) => {
+ const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
if (!thumbnail) return false
if (keepOnlyOwned) {
}
}
-async function doesActorImageExist (file: string) {
- const image = await ActorImageModel.loadByName(file)
+async function doesActorImageExist (filePath: string) {
+ const image = await ActorImageModel.loadByName(basename(filePath))
return !!image
}
-async function doesRedundancyExist (file: string) {
- const uuid = getUUIDFromFilename(file)
- const video = await VideoModel.loadWithFiles(uuid)
-
- if (!video) return false
-
- const isPlaylist = file.includes('.') === false
+async function doesRedundancyExist (filePath: string) {
+ const isPlaylist = (await stat(filePath)).isDirectory()
if (isPlaylist) {
+ const uuid = getUUIDFromFilename(filePath)
+ const video = await VideoModel.loadWithFiles(uuid)
+ if (!video) return false
+
const p = video.getHLSPlaylist()
if (!p) return false
return !!redundancy
}
- const resolution = parseInt(file.split('-')[5], 10)
- if (isNaN(resolution)) {
- console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
- return true
- }
-
- const videoFile = video.getWebTorrentFile(resolution)
- if (!videoFile) {
- console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
- return true
- }
+ const file = await VideoFileModel.loadByFilename(basename(filePath))
+ if (!file) return false
- const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
+ const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
return !!redundancy
}
import { VideoCommentModel } from '../server/models/video/video-comment'
import { AccountModel } from '../server/models/account/account'
import { VideoChannelModel } from '../server/models/video/video-channel'
-import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
import { initDatabaseModels } from '../server/initializers/database'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getServerActor } from '@server/models/application/application'
for (const file of video.VideoFiles) {
console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
await createTorrentAndSetInfoHash(video, file)
+
+ await file.save()
}
- for (const playlist of video.VideoStreamingPlaylists) {
- playlist.playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
- playlist.segmentsSha256Url = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive)
+ const playlist = video.getHLSPlaylist()
+ for (const file of (playlist?.VideoFiles || [])) {
+ console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
+
+ await createTorrentAndSetInfoHash(video, file)
- await playlist.save()
+ await file.save()
}
}
}
})
createTorrentFederate(video, videoFile)
+ .then(() => {
+ if (video.state !== VideoState.TO_TRANSCODE) return
- if (video.state === VideoState.TO_TRANSCODE) {
- await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
- }
+ return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
+ })
+ .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
return refreshedFile.save()
}
-function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void {
+function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile) {
// Create the torrent file in async way because it could be long
- createTorrentAndSetInfoHashAsync(video, videoFile)
+ return createTorrentAndSetInfoHashAsync(video, videoFile)
.catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) }))
.then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
.then(refreshedVideo => {
import * as retry from 'async/retry'
import * as Bluebird from 'bluebird'
-import { QueryTypes, Transaction } from 'sequelize'
+import { BindOrReplacements, QueryTypes, Transaction } from 'sequelize'
import { Model } from 'sequelize-typescript'
import { sequelizeTypescript } from '@server/initializers/database'
import { logger } from './logger'
})
}
-function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> (
+function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> (
fromDatabase: T[],
- newModels: T[],
- t: Transaction
+ newModels: T[]
) {
return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
- .map(f => f.destroy({ transaction: t }))
+}
+
+function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
+ return Promise.all(models.map(f => f.destroy({ transaction })))
}
// Sequelize always skip the update if we only update updatedAt field
// ---------------------------------------------------------------------------
+function doesExist (query: string, bind?: BindOrReplacements) {
+ const options = {
+ type: QueryTypes.SELECT as QueryTypes.SELECT,
+ bind,
+ raw: true
+ }
+
+ return sequelizeTypescript.query(query, options)
+ .then(results => results.length === 1)
+}
+
+// ---------------------------------------------------------------------------
+
export {
resetSequelizeInstance,
retryTransactionWrapper,
transactionRetryer,
updateInstanceWithAnother,
afterCommitIfTransaction,
- deleteNonExistingModels,
+ filterNonExistingModels,
+ deleteAllModels,
setAsUpdated,
- runInReadCommittedTransaction
+ runInReadCommittedTransaction,
+ doesExist
}
async function getLiveTranscodingCommand (options: {
rtmpUrl: string
+
outPath: string
+ masterPlaylistName: string
+
resolutions: number[]
fps: number
availableEncoders: AvailableEncoders
profile: string
}) {
- const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options
+ const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile, masterPlaylistName } = options
const input = rtmpUrl
const command = getFFmpeg(input, 'live')
command.complexFilter(complexFilter)
- addDefaultLiveHLSParams(command, outPath)
+ addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
command.outputOption('-var_stream_map', varStreamMap.join(' '))
return command
}
-function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
+function getLiveMuxingCommand (rtmpUrl: string, outPath: string, masterPlaylistName: string) {
const command = getFFmpeg(rtmpUrl, 'live')
command.outputOption('-c:v copy')
command.outputOption('-map 0:a?')
command.outputOption('-map 0:v?')
- addDefaultLiveHLSParams(command, outPath)
+ addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
return command
}
}
}
-function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) {
+function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, masterPlaylistName: string) {
command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
command.outputOption('-hls_flags delete_segments+independent_segments')
command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
- command.outputOption('-master_pl_name master.m3u8')
+ command.outputOption('-master_pl_name ' + masterPlaylistName)
command.outputOption(`-f hls`)
command.output(join(outPath, '%v.m3u8'))
await writeFile(torrentPath, torrent)
+ // Remove old torrent file if it existed
+ if (videoFile.hasTorrent()) {
+ await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
+ }
+
const parsedTorrent = parseTorrent(torrent)
videoFile.infoHash = parsedTorrent.infoHash
videoFile.torrentFilename = torrentFilename
// ---------------------------------------------------------------------------
-const LAST_MIGRATION_VERSION = 650
+const LAST_MIGRATION_VERSION = 655
// ---------------------------------------------------------------------------
--- /dev/null
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+ db: any
+}): Promise<void> {
+ {
+ for (const column of [ 'playlistUrl', 'segmentsSha256Url' ]) {
+ const data = {
+ type: Sequelize.STRING,
+ allowNull: true,
+ defaultValue: null
+ }
+
+ await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
+ }
+ }
+
+ {
+ await utils.sequelize.query(
+ `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = NULL, "segmentsSha256Url" = NULL ` +
+ `WHERE "videoId" IN (SELECT id FROM video WHERE remote IS FALSE)`
+ )
+ }
+
+ {
+ for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
+ const data = {
+ type: Sequelize.STRING,
+ allowNull: true,
+ defaultValue: null
+ }
+
+ await utils.queryInterface.addColumn('videoStreamingPlaylist', column, data)
+ }
+ }
+
+ {
+ await utils.sequelize.query(
+ `UPDATE "videoStreamingPlaylist" SET "playlistFilename" = 'master.m3u8', "segmentsSha256Filename" = 'segments-sha256.json'`
+ )
+ }
+
+ {
+ for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
+ const data = {
+ type: Sequelize.STRING,
+ allowNull: false,
+ defaultValue: null
+ }
+
+ await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
+ }
+ }
+}
+
+function down (options) {
+ throw new Error('Not implemented.')
+}
+
+export {
+ up,
+ down
+}
import { Transaction } from 'sequelize/types'
import { checkUrlsSameHost } from '@server/helpers/activitypub'
-import { deleteNonExistingModels } from '@server/helpers/database-utils'
+import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
import { logger, LoggerTagsFn } from '@server/helpers/logger'
import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
import { setVideoTags } from '@server/lib/video'
const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
// Remove video files that do not exist anymore
- const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t)
- await Promise.all(destroyTasks)
+ await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t)
// Update or add other one
const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
// Remove video playlists that do not exist anymore
- const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t)
- await Promise.all(destroyTasks)
+ await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)
video.VideoStreamingPlaylists = []
for (const playlistAttributes of streamingPlaylistAttributes) {
-
const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
streamingPlaylistModel.Video = video
const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
- const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
- await Promise.all(destroyTasks)
+ await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t)
// Update or add other one
const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
import { getExtFromMimetype } from '@server/helpers/video'
import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
import { generateTorrentFileName } from '@server/lib/video-paths'
+import { VideoCaptionModel } from '@server/models/video/video-caption'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { FilteredModelAttributes } from '@server/types'
-import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
+import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
import {
ActivityHashTagObject,
ActivityMagnetUrlObject,
VideoPrivacy,
VideoStreamingPlaylistType
} from '@shared/models'
-import { VideoCaptionModel } from '@server/models/video/video-caption'
function getThumbnailFromIcons (videoObject: VideoObject) {
let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
const resolution = fileUrl.height
- const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
- const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
+ const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id
+ const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null
const attribute = {
extname,
const attribute = {
type: VideoStreamingPlaylistType.HLS,
+
+ playlistFilename: basename(playlistUrlObject.href),
playlistUrl: playlistUrlObject.href,
+
+ segmentsSha256Filename: basename(segmentsSha256UrlObject.href),
segmentsSha256Url: segmentsSha256UrlObject.href,
+
p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
videoId: video.id,
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
import { flatten, uniq } from 'lodash'
import { basename, dirname, join } from 'path'
-import { MVideoWithFile } from '@server/types/models'
+import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
import { sha256 } from '../helpers/core-utils'
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
import { logger } from '../helpers/logger'
import { sequelizeTypescript } from '../initializers/database'
import { VideoFileModel } from '../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
-import { getVideoFilePath } from './video-paths'
+import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths'
async function updateStreamingPlaylistsInfohashesIfNeeded () {
const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
await sequelizeTypescript.transaction(async t => {
const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
- playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
+ playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
await playlist.save({ transaction: t })
})
}
}
-async function updateMasterHLSPlaylist (video: MVideoWithFile) {
+async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
+
const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
- const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
- const streamingPlaylist = video.getHLSPlaylist()
- for (const file of streamingPlaylist.VideoFiles) {
- const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)
+ const masterPlaylistPath = join(directory, playlist.playlistFilename)
+
+ for (const file of playlist.VideoFiles) {
+ const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
// If we did not generated a playlist for this resolution, skip
const filePlaylistPath = join(directory, playlistFilename)
if (await pathExists(filePlaylistPath) === false) continue
- const videoFilePath = getVideoFilePath(streamingPlaylist, file)
+ const videoFilePath = getVideoFilePath(playlist, file)
const size = await getVideoStreamSize(videoFilePath)
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
}
-async function updateSha256VODSegments (video: MVideoWithFile) {
+async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
const json: { [filename: string]: { [range: string]: string } } = {}
const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
- const hlsPlaylist = video.getHLSPlaylist()
// For all the resolutions available for this video
- for (const file of hlsPlaylist.VideoFiles) {
+ for (const file of playlist.VideoFiles) {
const rangeHashes: { [range: string]: string } = {}
- const videoPath = getVideoFilePath(hlsPlaylist, file)
- const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
+ const videoPath = getVideoFilePath(playlist, file)
+ const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
// Maybe the playlist is not generated for this resolution yet
- if (!await pathExists(playlistPath)) continue
+ if (!await pathExists(resolutionPlaylistPath)) continue
- const playlistContent = await readFile(playlistPath)
+ const playlistContent = await readFile(resolutionPlaylistPath)
const ranges = getRangesFromPlaylist(playlistContent.toString())
const fd = await open(videoPath, 'r')
json[videoFilename] = rangeHashes
}
- const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+ const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json)
}
if (currentVideoFile) {
// Remove old file and old torrent
- await video.removeFile(currentVideoFile)
- await currentVideoFile.removeTorrent()
+ await video.removeFileAndTorrent(currentVideoFile)
// Remove the old video file from the array
video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
import { generateVideoMiniature } from '@server/lib/thumbnail'
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
import { publishAndFederateIfNeeded } from '@server/lib/video'
-import { getHLSDirectory } from '@server/lib/video-paths'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths'
import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoLiveModel } from '@server/models/video/video-live'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
-import { MVideo, MVideoLive } from '@server/types/models'
+import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models'
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
import { logger } from '../../../helpers/logger'
return cleanupLive(video, streamingPlaylist)
}
- return saveLive(video, live)
+ return saveLive(video, live, streamingPlaylist)
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
-async function saveLive (video: MVideo, live: MVideoLive) {
+async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
const hlsDirectory = getHLSDirectory(video, false)
const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
const rootFiles = await readdir(hlsDirectory)
const playlistFiles = rootFiles.filter(file => {
- return file.endsWith('.m3u8') && file !== 'master.m3u8'
+ return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
})
await cleanupLiveFiles(hlsDirectory)
const hlsPlaylist = videoWithFiles.getHLSPlaylist()
await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
+
+ // Reset playlist
hlsPlaylist.VideoFiles = []
+ hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
+ hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
+ await hlsPlaylist.save()
let durationDone = false
if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
// Remove webtorrent files if not enabled
for (const file of video.VideoFiles) {
- await video.removeFile(file)
- await file.removeTorrent()
+ await video.removeFileAndTorrent(file)
await file.destroy()
}
import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
-import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
+import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME } from '@server/initializers/constants'
import { UserModel } from '@server/models/user/user'
import { VideoModel } from '@server/models/video/video'
import { VideoLiveModel } from '@server/models/video/video-live'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
-import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
+import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
import { federateVideoIfNeeded } from '../activitypub/videos'
import { JobQueue } from '../job-queue'
import { PeerTubeSocket } from '../peertube-socket'
+import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths'
import { LiveQuotaStore } from './live-quota-store'
import { LiveSegmentShaStore } from './live-segment-sha-store'
import { cleanupLive } from './live-utils'
return resolutionsEnabled.concat([ originResolution ])
}
- private async createLivePlaylist (video: MVideo, allResolutions: number[]) {
- const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
- const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
- videoId: video.id,
- playlistUrl,
- segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
- p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
- p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
+ private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise<MStreamingPlaylistVideo> {
+ const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
- type: VideoStreamingPlaylistType.HLS
- }, { returning: true }) as [ MStreamingPlaylist, boolean ]
+ playlist.playlistFilename = generateHLSMasterPlaylistFilename(true)
+ playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true)
- return Object.assign(videoStreamingPlaylist, { Video: video })
+ playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+ playlist.type = VideoStreamingPlaylistType.HLS
+
+ playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
+
+ return playlist.save()
}
static get Instance () {
this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
? await getLiveTranscodingCommand({
rtmpUrl: this.rtmpUrl,
+
outPath,
+ masterPlaylistName: this.streamingPlaylist.playlistFilename,
+
resolutions: this.allResolutions,
fps: this.fps,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.LIVE.TRANSCODING.PROFILE
})
- : getLiveMuxingCommand(this.rtmpUrl, outPath)
+ : getLiveMuxingCommand(this.rtmpUrl, outPath, this.streamingPlaylist.playlistFilename)
logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
}
private watchMasterFile (outPath: string) {
- this.masterWatcher = chokidar.watch(outPath + '/master.m3u8')
+ this.masterWatcher = chokidar.watch(outPath + '/' + this.streamingPlaylist.playlistFilename)
this.masterWatcher.on('add', async () => {
this.emit('master-playlist-created', { videoId: this.videoId })
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
- await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
+ const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
+ await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
expiresOn,
await sendCreateCacheFile(serverActor, video, createdModel)
- logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
+ logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url)
}
private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) {
if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
- return `${object.VideoStreamingPlaylist.playlistUrl}`
+ return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
}
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {
import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
import { logger } from '../../helpers/logger'
import { CONFIG } from '../../initializers/config'
-import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants'
+import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
import { VideoFileModel } from '../../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
-import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths'
+import {
+ generateHLSMasterPlaylistFilename,
+ generateHlsSha256SegmentsFilename,
+ generateHLSVideoFilename,
+ generateWebTorrentVideoFilename,
+ getHlsResolutionPlaylistFilename,
+ getVideoFilePath
+} from '../video-paths'
import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
/**
await ensureDir(videoTranscodedBasePath)
const videoFilename = generateHLSVideoFilename(resolution)
- const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)
- const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename)
+ const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
+ const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
const transcodeOptions = {
type,
inputPath,
- outputPath: playlistFileTranscodePath,
+ outputPath: resolutionPlaylistFileTranscodePath,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
await transcode(transcodeOptions)
- const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
-
// Create or update the playlist
- const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
- videoId: video.id,
- playlistUrl,
- segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
- p2pMediaLoaderInfohashes: [],
- p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
+ const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
+
+ if (!playlist.playlistFilename) {
+ playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
+ }
+
+ if (!playlist.segmentsSha256Filename) {
+ playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
+ }
+
+ playlist.p2pMediaLoaderInfohashes = []
+ playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
- type: VideoStreamingPlaylistType.HLS
- }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ]
- videoStreamingPlaylist.Video = video
+ playlist.type = VideoStreamingPlaylistType.HLS
+
+ await playlist.save()
// Build the new playlist file
const extname = extnameUtil(videoFilename)
size: 0,
filename: videoFilename,
fps: -1,
- videoStreamingPlaylistId: videoStreamingPlaylist.id
+ videoStreamingPlaylistId: playlist.id
})
- const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile)
+ const videoFilePath = getVideoFilePath(playlist, newVideoFile)
// Move files from tmp transcoded directory to the appropriate place
const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
await ensureDir(baseHlsDirectory)
// Move playlist file
- const playlistPath = join(baseHlsDirectory, playlistFilename)
- await move(playlistFileTranscodePath, playlistPath, { overwrite: true })
+ const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename)
+ await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
// Move video file
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
newVideoFile.fps = await getVideoFileFPS(videoFilePath)
newVideoFile.metadata = await getMetadataFromFile(videoFilePath)
- await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile)
+ await createTorrentAndSetInfoHash(playlist, newVideoFile)
await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
- videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles')
- videoStreamingPlaylist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(
- playlistUrl, videoStreamingPlaylist.VideoFiles
- )
- await videoStreamingPlaylist.save()
+ const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
+ playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
+ playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
+
+ await playlist.save()
- video.setHLSPlaylist(videoStreamingPlaylist)
+ video.setHLSPlaylist(playlist)
- await updateMasterHLSPlaylist(video)
- await updateSha256VODSegments(video)
+ await updateMasterHLSPlaylist(video, playlistWithFiles)
+ await updateSha256VODSegments(video, playlistWithFiles)
- return playlistPath
+ return resolutionPlaylistPath
}
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
import { buildUUID } from '@server/helpers/uuid'
+import { removeFragmentedMP4Ext } from '@shared/core-utils'
// ################## Video file name ##################
function generateWebTorrentVideoFilename (resolution: number, extname: string) {
- const uuid = buildUUID()
-
- return uuid + '-' + resolution + extname
+ return buildUUID() + '-' + resolution + extname
}
function generateHLSVideoFilename (resolution: number) {
- const uuid = buildUUID()
-
- return `${uuid}-${resolution}-fragmented.mp4`
+ return `${buildUUID()}-${resolution}-fragmented.mp4`
}
function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) {
return join(baseDir, video.uuid)
}
+function getHlsResolutionPlaylistFilename (videoFilename: string) {
+ // Video file name already contain resolution
+ return removeFragmentedMP4Ext(videoFilename) + '.m3u8'
+}
+
+function generateHLSMasterPlaylistFilename (isLive = false) {
+ if (isLive) return 'master.m3u8'
+
+ return buildUUID() + '-master.m3u8'
+}
+
+function generateHlsSha256SegmentsFilename (isLive = false) {
+ if (isLive) return 'segments-sha256.json'
+
+ return buildUUID() + '-segments-sha256.json'
+}
+
// ################## Torrents ##################
function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) {
getTorrentFilePath,
getHLSDirectory,
+ generateHLSMasterPlaylistFilename,
+ generateHlsSha256SegmentsFilename,
+ getHlsResolutionPlaylistFilename,
getLocalVideoFileMetadataUrl,
import { TagModel } from '@server/models/video/tag'
import { VideoModel } from '@server/models/video/video'
import { FilteredModelAttributes } from '@server/types'
-import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
+import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
import { federateVideoIfNeeded } from './activitypub/videos'
import { JobQueue } from './job-queue/job-queue'
}
}
-async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) {
+async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
let dataInput: VideoTranscodingPayload
if (videoFile.isAudio()) {
UpdatedAt
} from 'sequelize-typescript'
import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
+import { doesExist } from '@server/helpers/database-utils'
import { getServerActor } from '@server/models/application/application'
-import { VideoModel } from '@server/models/video/video'
import {
MActorFollowActorsDefault,
MActorFollowActorsDefaultSubscription,
static isFollowedBy (actorId: number, followerActorId: number) {
const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
- const options = {
- type: QueryTypes.SELECT as QueryTypes.SELECT,
- bind: { actorId, followerActorId },
- raw: true
- }
- return VideoModel.sequelize.query(query, options)
- .then(results => results.length === 1)
+ return doesExist(query, { actorId, followerActorId })
}
static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise<MActorFollowActorsDefault> {
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
- videoFile.Video.removeFile(videoFile, true)
- .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
+ videoFile.Video.removeFileAndTorrent(videoFile, true)
+ .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
}
if (instance.videoStreamingPlaylistId) {
return {
id: playlist.id,
type: playlist.type,
- playlistUrl: playlist.playlistUrl,
- segmentsSha256Url: playlist.segmentsSha256Url,
+ playlistUrl: playlist.getMasterPlaylistUrl(video),
+ segmentsSha256Url: playlist.getSha256SegmentsUrl(video),
redundancies,
files
}
type: 'Link',
name: 'sha256',
mediaType: 'application/json' as 'application/json',
- href: playlist.segmentsSha256Url
+ href: playlist.getSha256SegmentsUrl(video)
})
addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
url.push({
type: 'Link',
mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
- href: playlist.playlistUrl,
+ href: playlist.getMasterPlaylistUrl(video),
tag
})
}
}
getStreamingPlaylistAttributes () {
- let playlistKeys = [ 'id', 'playlistUrl', 'type' ]
+ let playlistKeys = [ 'id', 'playlistUrl', 'playlistFilename', 'type' ]
if (this.mode === 'get') {
playlistKeys = playlistKeys.concat([
'p2pMediaLoaderInfohashes',
'p2pMediaLoaderPeerVersion',
+ 'segmentsSha256Filename',
'segmentsSha256Url',
'videoId',
'createdAt',
import { remove } from 'fs-extra'
import * as memoizee from 'memoizee'
import { join } from 'path'
-import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
+import { FindOptions, Op, Transaction } from 'sequelize'
import {
AllowNull,
BelongsTo,
import { Where } from 'sequelize/types/lib/utils'
import validator from 'validator'
import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
+import { doesExist } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { extractVideo } from '@server/helpers/video'
import { getTorrentFilePath } from '@server/lib/video-paths'
static doesInfohashExist (infoHash: string) {
const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
- const options = {
- type: QueryTypes.SELECT as QueryTypes.SELECT,
- bind: { infoHash },
- raw: true
- }
- return VideoModel.sequelize.query(query, options)
- .then(results => results.length === 1)
+ return doesExist(query, { infoHash })
}
static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
return !!videoFile
}
+ static async doesOwnedTorrentFileExist (filename: string) {
+ const query = 'SELECT 1 FROM "videoFile" ' +
+ 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
+ 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
+ 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
+ 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
+
+ return doesExist(query, { filename })
+ }
+
+ static async doesOwnedWebTorrentVideoFileExist (filename: string) {
+ const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
+ 'WHERE "filename" = $filename LIMIT 1'
+
+ return doesExist(query, { filename })
+ }
+
+ static loadByFilename (filename: string) {
+ const query = {
+ where: {
+ filename
+ }
+ }
+
+ return VideoFileModel.findOne(query)
+ }
+
static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
const query = {
where: {
}
getFileDownloadUrl (video: MVideoWithHost) {
- const basePath = this.isHLS()
- ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS
- : STATIC_DOWNLOAD_PATHS.VIDEOS
- const path = join(basePath, this.filename)
+ const path = this.isHLS()
+ ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
+ : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
if (video.isOwned()) return WEBSERVER.URL + path
import * as memoizee from 'memoizee'
import { join } from 'path'
-import { Op, QueryTypes } from 'sequelize'
+import { Op } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { doesExist } from '@server/helpers/database-utils'
import { VideoFileModel } from '@server/models/video/video-file'
-import { MStreamingPlaylist } from '@server/types/models'
+import { MStreamingPlaylist, MVideo } from '@server/types/models'
+import { AttributesOnly } from '@shared/core-utils'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { sha1 } from '../../helpers/core-utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { isArrayOf } from '../../helpers/custom-validators/misc'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants'
+import {
+ CONSTRAINTS_FIELDS,
+ MEMOIZE_LENGTH,
+ MEMOIZE_TTL,
+ P2P_MEDIA_LOADER_PEER_VERSION,
+ STATIC_PATHS,
+ WEBSERVER
+} from '../../initializers/constants'
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { throwIfNotValid } from '../utils'
import { VideoModel } from './video'
-import { AttributesOnly } from '@shared/core-utils'
@Table({
tableName: 'videoStreamingPlaylist',
type: VideoStreamingPlaylistType
@AllowNull(false)
- @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
+ @Column
+ playlistFilename: string
+
+ @AllowNull(true)
+ @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
playlistUrl: string
p2pMediaLoaderPeerVersion: number
@AllowNull(false)
- @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
+ @Column
+ segmentsSha256Filename: string
+
+ @AllowNull(true)
+ @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true))
@Column
segmentsSha256Url: string
static doesInfohashExist (infoHash: string) {
const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
- const options = {
- type: QueryTypes.SELECT as QueryTypes.SELECT,
- bind: { infoHash },
- raw: true
- }
- return VideoModel.sequelize.query<object>(query, options)
- .then(results => results.length === 1)
+ return doesExist(query, { infoHash })
}
static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) {
p2pMediaLoaderPeerVersion: {
[Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
}
- }
+ },
+ include: [
+ {
+ model: VideoModel.unscoped(),
+ required: true
+ }
+ ]
}
return VideoStreamingPlaylistModel.findAll(query)
return VideoStreamingPlaylistModel.findByPk(id, options)
}
- static loadHLSPlaylistByVideo (videoId: number) {
+ static loadHLSPlaylistByVideo (videoId: number): Promise<MStreamingPlaylist> {
const options = {
where: {
type: VideoStreamingPlaylistType.HLS,
return VideoStreamingPlaylistModel.findOne(options)
}
- static getHlsPlaylistFilename (resolution: number) {
- return resolution + '.m3u8'
- }
+ static async loadOrGenerate (video: MVideo) {
+ let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
+ if (!playlist) playlist = new VideoStreamingPlaylistModel()
- static getMasterHlsPlaylistFilename () {
- return 'master.m3u8'
+ return Object.assign(playlist, { videoId: video.id, Video: video })
}
- static getHlsSha256SegmentsFilename () {
- return 'segments-sha256.json'
- }
+ assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
+ const masterPlaylistUrl = this.getMasterPlaylistUrl(video)
- static getHlsMasterPlaylistStaticPath (videoUUID: string) {
- return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
+ this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
}
- static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
- return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
+ getMasterPlaylistUrl (video: MVideo) {
+ if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
+
+ return this.playlistUrl
}
- static getHlsSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
- if (isLive) return join('/live', 'segments-sha256', videoUUID)
+ getSha256SegmentsUrl (video: MVideo) {
+ if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
- return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
+ return this.segmentsSha256Url
}
getStringType () {
return this.type === other.type &&
this.videoId === other.videoId
}
+
+ private getMasterPlaylistStaticPath (videoUUID: string) {
+ return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
+ }
+
+ private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
+ if (isLive) return join('/live', 'segments-sha256', videoUUID)
+
+ return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
+ }
}
// Remove physical files and torrents
instance.VideoFiles.forEach(file => {
- tasks.push(instance.removeFile(file))
- tasks.push(file.removeTorrent())
+ tasks.push(instance.removeFileAndTorrent(file))
})
// Remove playlists file
.concat(toAdd)
}
- removeFile (videoFile: MVideoFile, isRedundancy = false) {
+ removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
const filePath = getVideoFilePath(this, videoFile, isRedundancy)
- return remove(filePath)
- .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
+
+ const promises: Promise<any>[] = [ remove(filePath) ]
+ if (!isRedundancy) promises.push(videoFile.removeTorrent())
+
+ return Promise.all(promises)
}
async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {
import * as chai from 'chai'
import { VideoPrivacy } from '@shared/models'
import {
- checkLiveCleanup,
+ checkLiveCleanupAfterSave,
cleanupTests,
ConfigCommand,
createMultipleServers,
expect(video.duration).to.be.greaterThan(0)
}
- await checkLiveCleanup(servers[0], videoId, resolutions)
+ await checkLiveCleanupAfterSave(servers[0], videoId, resolutions)
}
async function waitUntilLivePublishedOnAllServers (videoId: string) {
import * as chai from 'chai'
import { FfmpegCommand } from 'fluent-ffmpeg'
import {
- checkLiveCleanup,
+ checkLiveCleanupAfterSave,
cleanupTests,
ConfigCommand,
createMultipleServers,
await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
// No resolutions saved since we did not save replay
- await checkLiveCleanup(servers[0], liveVideoUUID, [])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
it('Should correctly terminate the stream on blacklist and delete the live', async function () {
await wait(5000)
await waitJobs(servers)
- await checkLiveCleanup(servers[0], liveVideoUUID, [])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
it('Should correctly terminate the stream on delete and delete the video', async function () {
await waitJobs(servers)
await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
- await checkLiveCleanup(servers[0], liveVideoUUID, [])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
})
})
it('Should have cleaned up the live files', async function () {
- await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
})
it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
await wait(5000)
await waitJobs(servers)
- await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
})
it('Should correctly terminate the stream on delete and delete the video', async function () {
await waitJobs(servers)
await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
- await checkLiveCleanup(servers[0], liveVideoUUID, [])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
})
import 'mocha'
import * as chai from 'chai'
-import { join } from 'path'
+import { basename, join } from 'path'
import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
import {
- checkLiveCleanup,
+ checkLiveCleanupAfterSave,
checkLiveSegmentHash,
checkResolutionsInMasterPlaylist,
cleanupTests,
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
+ // We should have generated random filenames
+ expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
+ expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
+
expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
for (const resolution of resolutions) {
expect(file.fps).to.be.approximately(30, 2)
}
- const filename = `${video.uuid}-${resolution}-fragmented.mp4`
+ const filename = basename(file.fileUrl)
+ expect(filename).to.not.contain(video.uuid)
+
const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
const probe = await ffprobePromise(segmentPath)
it('Should correctly have cleaned up the live files', async function () {
this.timeout(30000)
- await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ])
+ await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
})
})
const { uuid } = await servers[0].videos.upload({ token: userAccessToken })
videoUUID = uuid
+ await waitJobs(servers)
+
await saveVideoInServers(servers, videoUUID)
}
-
- await waitJobs(servers)
})
it('Should be able to update my display name', async function () {
const size = 1000
+ // Content length check seems to have changed in v16
+ const expectedStatus = process.version.startsWith('v16')
+ ? HttpStatusCode.CONFLICT_409
+ : HttpStatusCode.BAD_REQUEST_400
+
const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}`
- await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size })
+ await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size })
await checkFileSize(uploadId, 0)
})
})
import 'mocha'
import * as chai from 'chai'
-import { join } from 'path'
+import { basename, join } from 'path'
+import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
import {
checkDirectoryIsEmpty,
checkResolutionsInMasterPlaylist,
} from '@shared/extra-utils'
import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
-import { uuidRegex } from '@shared/core-utils'
-import { basename } from 'path/posix'
const expect = chai.expect
// Check resolution playlists
{
for (const resolution of resolutions) {
+ const file = hlsFiles.find(f => f.resolution.id === resolution)
+ const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
+
const subPlaylist = await server.streamingPlaylists.get({
- url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8`
+ url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
})
- const file = hlsFiles.find(f => f.resolution.id === resolution)
expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
expect(subPlaylist).to.contain(basename(file.fileUrl))
}
import 'mocha'
import * as chai from 'chai'
-import { join } from 'path'
import {
cleanupTests,
createMultipleServers,
expect(file.size).to.be.below(8000000)
- const path = servers[0].servers.buildDirectory(join('videos', video.uuid + '-' + file.resolution.id + '.mp4'))
+ const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl)
const bitrate = await getVideoFileBitrate(path)
const fps = await getVideoFileFPS(path)
const resolution = await getVideoFileResolution(path)
}
}
-async function assertCountAreOkay (servers: PeerTubeServer[]) {
+async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) {
for (const server of servers) {
const videosCount = await countFiles(server, 'videos')
expect(videosCount).to.equal(8)
const avatarsCount = await countFiles(server, 'avatars')
expect(avatarsCount).to.equal(2)
}
+
+ // When we'll prune HLS directories too
+ // const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/')
+ // expect(hlsRootCount).to.equal(2)
+
+ // const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID)
+ // expect(hlsCount).to.equal(10)
}
describe('Test prune storage scripts', function () {
let servers: PeerTubeServer[]
const badNames: { [directory: string]: string[] } = {}
+ let videoServer2UUID: string
+
before(async function () {
this.timeout(120000)
for (const server of servers) {
await server.videos.upload({ attributes: { name: 'video 1' } })
- await server.videos.upload({ attributes: { name: 'video 2' } })
+
+ const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
+ if (server.serverNumber === 2) videoServer2UUID = uuid
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
})
it('Should have the files on the disk', async function () {
- await assertCountAreOkay(servers)
+ await assertCountAreOkay(servers, videoServer2UUID)
})
it('Should create some dirty files', async function () {
badNames['avatars'] = [ n1, n2 ]
}
+
+ // When we'll prune HLS directories too
+ // {
+ // const directory = join('streaming-playlists', 'hls')
+ // const base = servers[1].servers.buildDirectory(directory)
+
+ // const n1 = buildUUID()
+ // await createFile(join(base, n1))
+ // badNames[directory] = [ n1 ]
+ // }
+
+ // {
+ // const directory = join('streaming-playlists', 'hls', videoServer2UUID)
+ // const base = servers[1].servers.buildDirectory(directory)
+ // const n1 = buildUUID() + '-240-fragmented-.mp4'
+ // const n2 = buildUUID() + '-master.m3u8'
+
+ // await createFile(join(base, n1))
+ // await createFile(join(base, n2))
+
+ // badNames[directory] = [ n1, n2 ]
+ // }
}
})
})
it('Should have removed files', async function () {
- await assertCountAreOkay(servers)
+ await assertCountAreOkay(servers, videoServer2UUID)
for (const directory of Object.keys(badNames)) {
for (const name of badNames[directory]) {
for (const video of data) {
const videoDetails = await server.videos.get({ id: video.id })
+ const files = videoDetails.files.concat(videoDetails.streamingPlaylists[0].files)
- expect(videoDetails.files).to.have.lengthOf(4)
+ expect(files).to.have.lengthOf(8)
- for (const file of videoDetails.files) {
+ for (const file of files) {
expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket')
- expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F')
+ expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2F')
- const torrent = await parseTorrentVideo(server, videoDetails.uuid, file.resolution.id)
+ const torrent = await parseTorrentVideo(server, file)
const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket')
expect(announceWS).to.not.be.undefined
const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce')
expect(announceHttp).to.not.be.undefined
- expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed')
+ expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/')
}
}
})
import 'mocha'
import { expect } from 'chai'
-import { join } from 'path'
import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
import {
cleanupTests,
const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid
await waitJobs([ server ])
- const path = server.servers.buildDirectory(join('videos', videoUUID + '-240.mp4'))
+ const video = await server.videos.get({ id: videoUUID })
+
+ const path = server.servers.buildWebTorrentFilePath(video.files[0].fileUrl)
const audioProbe = await getAudioStream(path)
expect(audioProbe.audioStream.codec_name).to.equal('opus')
PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo {
- return !!(value as MStreamingPlaylist).playlistUrl
+ return !!(value as MStreamingPlaylist).videoId
}
export const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
+export function removeFragmentedMP4Ext (path: string) {
+ return path.replace(/-fragmented.mp4$/i, '')
+}
import { readFile } from 'fs-extra'
import * as parseTorrent from 'parse-torrent'
-import { join } from 'path'
+import { basename, join } from 'path'
import * as WebTorrent from 'webtorrent'
+import { VideoFile } from '@shared/models'
import { PeerTubeServer } from '../server'
let webtorrent: WebTorrent.Instance
return new Promise<WebTorrent.Torrent>(res => webtorrent.add(torrent, res))
}
-async function parseTorrentVideo (server: PeerTubeServer, videoUUID: string, resolution: number) {
- const torrentName = videoUUID + '-' + resolution + '.torrent'
+async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) {
+ const torrentName = basename(file.torrentUrl)
const torrentPath = server.servers.buildDirectory(join('torrents', torrentName))
const data = await readFile(torrentPath)
import { exec } from 'child_process'
import { copy, ensureDir, readFile, remove } from 'fs-extra'
-import { join } from 'path'
-import { basename } from 'path/posix'
+import { basename, join } from 'path'
import { root } from '@server/helpers/core-utils'
import { HttpStatusCode } from '@shared/models'
import { getFileSize, isGithubCI, wait } from '../miscs'
}
}
-async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
+async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
const basePath = server.servers.buildDirectory('streaming-playlists')
const hlsPath = join(basePath, 'hls', videoUUID)
expect(files).to.have.lengthOf(resolutions.length * 2 + 2)
for (const resolution of resolutions) {
- expect(files).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`)
- expect(files).to.contain(`${resolution}.m3u8`)
+ const fragmentedFile = files.find(f => f.endsWith(`-${resolution}-fragmented.mp4`))
+ expect(fragmentedFile).to.exist
+
+ const playlistFile = files.find(f => f.endsWith(`${resolution}.m3u8`))
+ expect(playlistFile).to.exist
}
- expect(files).to.contain('master.m3u8')
- expect(files).to.contain('segments-sha256.json')
+ const masterPlaylistFile = files.find(f => f.endsWith('-master.m3u8'))
+ expect(masterPlaylistFile).to.exist
+
+ const shaFile = files.find(f => f.endsWith('-segments-sha256.json'))
+ expect(shaFile).to.exist
}
export {
testFfmpegStreamError,
stopFfmpeg,
waitUntilLivePublishedOnAllServers,
- checkLiveCleanup
+ checkLiveCleanupAfterSave
}
import { expect } from 'chai'
import { basename } from 'path'
import { sha256 } from '@server/helpers/core-utils'
+import { removeFragmentedMP4Ext } from '@shared/core-utils'
import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models'
import { PeerTubeServer } from '../server'
const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options
const command = server.streamingPlaylists
- const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8` })
-
const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
const videoName = basename(file.fileUrl)
+ const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` })
+
const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
const length = parseInt(matches[1], 10)
projects / github / Chocobozzz / PeerTube.git / commitdiff
