From 764b1a14fc494f2cfd7ea590d2f07b01df65c7ad Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 23 Jul 2021 11:20:00 +0200 Subject: Use random names for VOD HLS playlists --- server/controllers/api/videos/upload.ts | 12 +-- server/helpers/database-utils.ts | 31 ++++++-- server/helpers/ffmpeg-utils.ts | 15 ++-- server/helpers/webtorrent.ts | 5 ++ server/initializers/constants.ts | 2 +- .../0655-streaming-playlist-filenames.ts | 66 +++++++++++++++++ .../activitypub/videos/shared/abstract-builder.ts | 12 +-- .../videos/shared/object-to-model-attributes.ts | 13 +++- server/lib/hls.ts | 35 ++++----- server/lib/job-queue/handlers/video-file-import.ts | 3 +- server/lib/job-queue/handlers/video-live-ending.ts | 15 ++-- server/lib/job-queue/handlers/video-transcoding.ts | 3 +- server/lib/live/live-manager.ts | 26 +++---- server/lib/live/shared/muxing-session.ts | 7 +- .../lib/schedulers/videos-redundancy-scheduler.ts | 7 +- server/lib/transcoding/video-transcoding.ts | 71 ++++++++++-------- server/lib/video-paths.ts | 29 ++++++-- server/lib/video.ts | 4 +- server/models/actor/actor-follow.ts | 10 +-- server/models/redundancy/video-redundancy.ts | 4 +- .../models/video/formatter/video-format-utils.ts | 8 +- server/models/video/sql/shared/video-tables.ts | 3 +- server/models/video/video-file.ts | 45 +++++++++--- server/models/video/video-streaming-playlist.ts | 85 ++++++++++++++-------- server/models/video/video.ts | 12 +-- server/tests/api/live/live-constraints.ts | 4 +- server/tests/api/live/live-save-replay.ts | 14 ++-- server/tests/api/live/live.ts | 14 +++- server/tests/api/users/users-multiple-servers.ts | 4 +- server/tests/api/videos/resumable-upload.ts | 7 +- server/tests/api/videos/video-hls.ts | 11 +-- server/tests/cli/optimize-old-videos.ts | 3 +- server/tests/cli/prune-storage.ts | 41 ++++++++++- server/tests/cli/update-host.ts | 11 +-- server/tests/plugins/plugin-transcoding.ts | 5 +- .../types/models/video/video-streaming-playlist.ts | 2 +- 36 files changed, 429 insertions(+), 210 deletions(-) create mode 100644 server/initializers/migrations/0655-streaming-playlist-filenames.ts (limited to 'server') diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 7792ae3fc..408f677ff 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -209,10 +209,12 @@ async function addVideo (options: { }) createTorrentFederate(video, videoFile) + .then(() => { + if (video.state !== VideoState.TO_TRANSCODE) return - if (video.state === VideoState.TO_TRANSCODE) { - await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) - } + return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) + }) + .catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) @@ -259,9 +261,9 @@ async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoF return refreshedFile.save() } -function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void { +function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile) { // Create the torrent file in async way because it could be long - createTorrentAndSetInfoHashAsync(video, videoFile) + return createTorrentAndSetInfoHashAsync(video, videoFile) .catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) })) .then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)) .then(refreshedVideo => { diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index cbd7aa401..422774022 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -1,6 +1,6 @@ import * as retry from 'async/retry' import * as Bluebird from 'bluebird' -import { QueryTypes, Transaction } from 'sequelize' +import { BindOrReplacements, QueryTypes, Transaction } from 'sequelize' import { Model } from 'sequelize-typescript' import { sequelizeTypescript } from '@server/initializers/database' import { logger } from './logger' @@ -84,13 +84,15 @@ function resetSequelizeInstance (instance: Model, savedFields: object) { }) } -function deleteNonExistingModels > ( +function filterNonExistingModels ( fromDatabase: T[], - newModels: T[], - t: Transaction + newModels: T[] ) { return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f))) - .map(f => f.destroy({ transaction: t })) +} + +function deleteAllModels > (models: T[], transaction: Transaction) { + return Promise.all(models.map(f => f.destroy({ transaction }))) } // Sequelize always skip the update if we only update updatedAt field @@ -121,13 +123,28 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) { // --------------------------------------------------------------------------- +function doesExist (query: string, bind?: BindOrReplacements) { + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + bind, + raw: true + } + + return sequelizeTypescript.query(query, options) + .then(results => results.length === 1) +} + +// --------------------------------------------------------------------------- + export { resetSequelizeInstance, retryTransactionWrapper, transactionRetryer, updateInstanceWithAnother, afterCommitIfTransaction, - deleteNonExistingModels, + filterNonExistingModels, + deleteAllModels, setAsUpdated, - runInReadCommittedTransaction + runInReadCommittedTransaction, + doesExist } diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 6f5a71b4a..9ad4b7f3b 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -212,14 +212,17 @@ async function transcode (options: TranscodeOptions) { async function getLiveTranscodingCommand (options: { rtmpUrl: string + outPath: string + masterPlaylistName: string + resolutions: number[] fps: number availableEncoders: AvailableEncoders profile: string }) { - const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options + const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile, masterPlaylistName } = options const input = rtmpUrl const command = getFFmpeg(input, 'live') @@ -301,14 +304,14 @@ async function getLiveTranscodingCommand (options: { command.complexFilter(complexFilter) - addDefaultLiveHLSParams(command, outPath) + addDefaultLiveHLSParams(command, outPath, masterPlaylistName) command.outputOption('-var_stream_map', varStreamMap.join(' ')) return command } -function getLiveMuxingCommand (rtmpUrl: string, outPath: string) { +function getLiveMuxingCommand (rtmpUrl: string, outPath: string, masterPlaylistName: string) { const command = getFFmpeg(rtmpUrl, 'live') command.outputOption('-c:v copy') @@ -316,7 +319,7 @@ function getLiveMuxingCommand (rtmpUrl: string, outPath: string) { command.outputOption('-map 0:a?') command.outputOption('-map 0:v?') - addDefaultLiveHLSParams(command, outPath) + addDefaultLiveHLSParams(command, outPath, masterPlaylistName) return command } @@ -371,12 +374,12 @@ function addDefaultEncoderParams (options: { } } -function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) { +function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, masterPlaylistName: string) { command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS) command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE) command.outputOption('-hls_flags delete_segments+independent_segments') command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) - command.outputOption('-master_pl_name master.m3u8') + command.outputOption('-master_pl_name ' + masterPlaylistName) command.outputOption(`-f hls`) command.output(join(outPath, '%v.m3u8')) diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index d8220ba9c..ecf63e93e 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -103,6 +103,11 @@ async function createTorrentAndSetInfoHash ( await writeFile(torrentPath, torrent) + // Remove old torrent file if it existed + if (videoFile.hasTorrent()) { + await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) + } + const parsedTorrent = parseTorrent(torrent) videoFile.infoHash = parsedTorrent.infoHash videoFile.torrentFilename = torrentFilename diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index ab59320eb..ee4503b2c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 650 +const LAST_MIGRATION_VERSION = 655 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0655-streaming-playlist-filenames.ts b/server/initializers/migrations/0655-streaming-playlist-filenames.ts new file mode 100644 index 000000000..9172a22c4 --- /dev/null +++ b/server/initializers/migrations/0655-streaming-playlist-filenames.ts @@ -0,0 +1,66 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + { + for (const column of [ 'playlistUrl', 'segmentsSha256Url' ]) { + const data = { + type: Sequelize.STRING, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data) + } + } + + { + await utils.sequelize.query( + `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = NULL, "segmentsSha256Url" = NULL ` + + `WHERE "videoId" IN (SELECT id FROM video WHERE remote IS FALSE)` + ) + } + + { + for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) { + const data = { + type: Sequelize.STRING, + allowNull: true, + defaultValue: null + } + + await utils.queryInterface.addColumn('videoStreamingPlaylist', column, data) + } + } + + { + await utils.sequelize.query( + `UPDATE "videoStreamingPlaylist" SET "playlistFilename" = 'master.m3u8', "segmentsSha256Filename" = 'segments-sha256.json'` + ) + } + + { + for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) { + const data = { + type: Sequelize.STRING, + allowNull: false, + defaultValue: null + } + + await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data) + } + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index e89c94bcd..f995fe637 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -1,6 +1,6 @@ import { Transaction } from 'sequelize/types' import { checkUrlsSameHost } from '@server/helpers/activitypub' -import { deleteNonExistingModels } from '@server/helpers/database-utils' +import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' import { logger, LoggerTagsFn } from '@server/helpers/logger' import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' import { setVideoTags } from '@server/lib/video' @@ -111,8 +111,7 @@ export abstract class APVideoAbstractBuilder { const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) // Remove video files that do not exist anymore - const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t) - await Promise.all(destroyTasks) + await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t) // Update or add other one const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) @@ -124,13 +123,11 @@ export abstract class APVideoAbstractBuilder { const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) // Remove video playlists that do not exist anymore - const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t) - await Promise.all(destroyTasks) + await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) video.VideoStreamingPlaylists = [] for (const playlistAttributes of streamingPlaylistAttributes) { - const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) streamingPlaylistModel.Video = video @@ -163,8 +160,7 @@ export abstract class APVideoAbstractBuilder { const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) - const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t) - await Promise.all(destroyTasks) + await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t) // Update or add other one const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index 85548428c..1fa16295d 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -7,10 +7,11 @@ import { logger } from '@server/helpers/logger' import { getExtFromMimetype } from '@server/helpers/video' import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' import { generateTorrentFileName } from '@server/lib/video-paths' +import { VideoCaptionModel } from '@server/models/video/video-caption' import { VideoFileModel } from '@server/models/video/video-file' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' import { FilteredModelAttributes } from '@server/types' -import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models' +import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models' import { ActivityHashTagObject, ActivityMagnetUrlObject, @@ -23,7 +24,6 @@ import { VideoPrivacy, VideoStreamingPlaylistType } from '@shared/models' -import { VideoCaptionModel } from '@server/models/video/video-caption' function getThumbnailFromIcons (videoObject: VideoObject) { let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) @@ -80,8 +80,8 @@ function getFileAttributesFromUrl ( const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) const resolution = fileUrl.height - const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id - const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null + const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id + const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null const attribute = { extname, @@ -130,8 +130,13 @@ function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: const attribute = { type: VideoStreamingPlaylistType.HLS, + + playlistFilename: basename(playlistUrlObject.href), playlistUrl: playlistUrlObject.href, + + segmentsSha256Filename: basename(segmentsSha256UrlObject.href), segmentsSha256Url: segmentsSha256UrlObject.href, + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, videoId: video.id, diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 212bd095b..32b02bc26 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -1,7 +1,7 @@ import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra' import { flatten, uniq } from 'lodash' import { basename, dirname, join } from 'path' -import { MVideoWithFile } from '@server/types/models' +import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models' import { sha256 } from '../helpers/core-utils' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' import { logger } from '../helpers/logger' @@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { getVideoFilePath } from './video-paths' +import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -22,27 +22,29 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () { await sequelizeTypescript.transaction(async t => { const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) - playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles) + playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + await playlist.save({ transaction: t }) }) } } -async function updateMasterHLSPlaylist (video: MVideoWithFile) { +async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) - const streamingPlaylist = video.getHLSPlaylist() - for (const file of streamingPlaylist.VideoFiles) { - const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution) + const masterPlaylistPath = join(directory, playlist.playlistFilename) + + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) // If we did not generated a playlist for this resolution, skip const filePlaylistPath = join(directory, playlistFilename) if (await pathExists(filePlaylistPath) === false) continue - const videoFilePath = getVideoFilePath(streamingPlaylist, file) + const videoFilePath = getVideoFilePath(playlist, file) const size = await getVideoStreamSize(videoFilePath) @@ -66,23 +68,22 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) { await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') } -async function updateSha256VODSegments (video: MVideoWithFile) { +async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) { const json: { [filename: string]: { [range: string]: string } } = {} const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) - const hlsPlaylist = video.getHLSPlaylist() // For all the resolutions available for this video - for (const file of hlsPlaylist.VideoFiles) { + for (const file of playlist.VideoFiles) { const rangeHashes: { [range: string]: string } = {} - const videoPath = getVideoFilePath(hlsPlaylist, file) - const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + const videoPath = getVideoFilePath(playlist, file) + const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename)) // Maybe the playlist is not generated for this resolution yet - if (!await pathExists(playlistPath)) continue + if (!await pathExists(resolutionPlaylistPath)) continue - const playlistContent = await readFile(playlistPath) + const playlistContent = await readFile(resolutionPlaylistPath) const ranges = getRangesFromPlaylist(playlistContent.toString()) const fd = await open(videoPath, 'r') @@ -98,7 +99,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) { json[videoFilename] = rangeHashes } - const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename()) + const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename) await outputJSON(outputPath, json) } diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 1783f206a..4d199f247 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -61,8 +61,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { if (currentVideoFile) { // Remove old file and old torrent - await video.removeFile(currentVideoFile) - await currentVideoFile.removeTorrent() + await video.removeFileAndTorrent(currentVideoFile) // Remove the old video file from the array video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 9eba41bf8..386ccdc7b 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -7,12 +7,12 @@ import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server import { generateVideoMiniature } from '@server/lib/thumbnail' import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding' import { publishAndFederateIfNeeded } from '@server/lib/video' -import { getHLSDirectory } from '@server/lib/video-paths' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths' import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MVideo, MVideoLive } from '@server/types/models' +import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models' import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' import { logger } from '../../../helpers/logger' @@ -43,7 +43,7 @@ async function processVideoLiveEnding (job: Bull.Job) { return cleanupLive(video, streamingPlaylist) } - return saveLive(video, live) + return saveLive(video, live, streamingPlaylist) } // --------------------------------------------------------------------------- @@ -54,14 +54,14 @@ export { // --------------------------------------------------------------------------- -async function saveLive (video: MVideo, live: MVideoLive) { +async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) { const hlsDirectory = getHLSDirectory(video, false) const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY) const rootFiles = await readdir(hlsDirectory) const playlistFiles = rootFiles.filter(file => { - return file.endsWith('.m3u8') && file !== 'master.m3u8' + return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename }) await cleanupLiveFiles(hlsDirectory) @@ -80,7 +80,12 @@ async function saveLive (video: MVideo, live: MVideoLive) { const hlsPlaylist = videoWithFiles.getHLSPlaylist() await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) + + // Reset playlist hlsPlaylist.VideoFiles = [] + hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename() + hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() + await hlsPlaylist.save() let durationDone = false diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index f5ba6f435..36d9594af 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -125,8 +125,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) { // Remove webtorrent files if not enabled for (const file of video.VideoFiles) { - await video.removeFile(file) - await file.removeTorrent() + await video.removeFileAndTorrent(file) await file.destroy() } diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index da764e009..f106d69fb 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -4,16 +4,17 @@ import { isTestInstance } from '@server/helpers/core-utils' import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants' +import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME } from '@server/initializers/constants' import { UserModel } from '@server/models/user/user' import { VideoModel } from '@server/models/video/video' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' +import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models' import { VideoState, VideoStreamingPlaylistType } from '@shared/models' import { federateVideoIfNeeded } from '../activitypub/videos' import { JobQueue } from '../job-queue' import { PeerTubeSocket } from '../peertube-socket' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths' import { LiveQuotaStore } from './live-quota-store' import { LiveSegmentShaStore } from './live-segment-sha-store' import { cleanupLive } from './live-utils' @@ -392,19 +393,18 @@ class LiveManager { return resolutionsEnabled.concat([ originResolution ]) } - private async createLivePlaylist (video: MVideo, allResolutions: number[]) { - const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) - const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ - videoId: video.id, - playlistUrl, - segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), - p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions), - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, + private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise { + const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video) - type: VideoStreamingPlaylistType.HLS - }, { returning: true }) as [ MStreamingPlaylist, boolean ] + playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) - return Object.assign(videoStreamingPlaylist, { Video: video }) + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + playlist.type = VideoStreamingPlaylistType.HLS + + playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) + + return playlist.save() } static get Instance () { diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts index 26467f060..709d6c615 100644 --- a/server/lib/live/shared/muxing-session.ts +++ b/server/lib/live/shared/muxing-session.ts @@ -112,13 +112,16 @@ class MuxingSession extends EventEmitter { this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED ? await getLiveTranscodingCommand({ rtmpUrl: this.rtmpUrl, + outPath, + masterPlaylistName: this.streamingPlaylist.playlistFilename, + resolutions: this.allResolutions, fps: this.fps, availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), profile: CONFIG.LIVE.TRANSCODING.PROFILE }) - : getLiveMuxingCommand(this.rtmpUrl, outPath) + : getLiveMuxingCommand(this.rtmpUrl, outPath, this.streamingPlaylist.playlistFilename) logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags) @@ -182,7 +185,7 @@ class MuxingSession extends EventEmitter { } private watchMasterFile (outPath: string) { - this.masterWatcher = chokidar.watch(outPath + '/master.m3u8') + this.masterWatcher = chokidar.watch(outPath + '/' + this.streamingPlaylist.playlistFilename) this.masterWatcher.on('add', async () => { this.emit('master-playlist-created', { videoId: this.videoId }) diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index b5a5eb697..103ab1fab 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -267,7 +267,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) - await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) + const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) + await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ expiresOn, @@ -282,7 +283,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { await sendCreateCacheFile(serverActor, video, createdModel) - logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url) + logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url) } private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { @@ -330,7 +331,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` - return `${object.VideoStreamingPlaylist.playlistUrl}` + return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` } private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index d70f7f474..d2a556360 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts @@ -10,11 +10,18 @@ import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils' import { logger } from '../../helpers/logger' import { CONFIG } from '../../initializers/config' -import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants' +import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants' import { VideoFileModel } from '../../models/video/video-file' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' -import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths' +import { + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + generateHLSVideoFilename, + generateWebTorrentVideoFilename, + getHlsResolutionPlaylistFilename, + getVideoFilePath +} from '../video-paths' import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' /** @@ -272,14 +279,14 @@ async function generateHlsPlaylistCommon (options: { await ensureDir(videoTranscodedBasePath) const videoFilename = generateHLSVideoFilename(resolution) - const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution) - const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename) + const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) + const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename) const transcodeOptions = { type, inputPath, - outputPath: playlistFileTranscodePath, + outputPath: resolutionPlaylistFileTranscodePath, availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(), profile: CONFIG.TRANSCODING.PROFILE, @@ -299,19 +306,23 @@ async function generateHlsPlaylistCommon (options: { await transcode(transcodeOptions) - const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) - // Create or update the playlist - const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({ - videoId: video.id, - playlistUrl, - segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive), - p2pMediaLoaderInfohashes: [], - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, + const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video) + + if (!playlist.playlistFilename) { + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + } + + if (!playlist.segmentsSha256Filename) { + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) + } + + playlist.p2pMediaLoaderInfohashes = [] + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - type: VideoStreamingPlaylistType.HLS - }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ] - videoStreamingPlaylist.Video = video + playlist.type = VideoStreamingPlaylistType.HLS + + await playlist.save() // Build the new playlist file const extname = extnameUtil(videoFilename) @@ -321,18 +332,18 @@ async function generateHlsPlaylistCommon (options: { size: 0, filename: videoFilename, fps: -1, - videoStreamingPlaylistId: videoStreamingPlaylist.id + videoStreamingPlaylistId: playlist.id }) - const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile) + const videoFilePath = getVideoFilePath(playlist, newVideoFile) // Move files from tmp transcoded directory to the appropriate place const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) await ensureDir(baseHlsDirectory) // Move playlist file - const playlistPath = join(baseHlsDirectory, playlistFilename) - await move(playlistFileTranscodePath, playlistPath, { overwrite: true }) + const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename) + await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true }) // Move video file await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true }) @@ -342,20 +353,20 @@ async function generateHlsPlaylistCommon (options: { newVideoFile.fps = await getVideoFileFPS(videoFilePath) newVideoFile.metadata = await getMetadataFromFile(videoFilePath) - await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) + await createTorrentAndSetInfoHash(playlist, newVideoFile) await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) - videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles') - videoStreamingPlaylist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes( - playlistUrl, videoStreamingPlaylist.VideoFiles - ) - await videoStreamingPlaylist.save() + const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo + playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles') + playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + + await playlist.save() - video.setHLSPlaylist(videoStreamingPlaylist) + video.setHLSPlaylist(playlist) - await updateMasterHLSPlaylist(video) - await updateSha256VODSegments(video) + await updateMasterHLSPlaylist(video, playlistWithFiles) + await updateSha256VODSegments(video, playlistWithFiles) - return playlistPath + return resolutionPlaylistPath } diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index b7068190c..1e4382108 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts @@ -4,19 +4,16 @@ import { CONFIG } from '@server/initializers/config' import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' import { buildUUID } from '@server/helpers/uuid' +import { removeFragmentedMP4Ext } from '@shared/core-utils' // ################## Video file name ################## function generateWebTorrentVideoFilename (resolution: number, extname: string) { - const uuid = buildUUID() - - return uuid + '-' + resolution + extname + return buildUUID() + '-' + resolution + extname } function generateHLSVideoFilename (resolution: number) { - const uuid = buildUUID() - - return `${uuid}-${resolution}-fragmented.mp4` + return `${buildUUID()}-${resolution}-fragmented.mp4` } function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { @@ -54,6 +51,23 @@ function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { return join(baseDir, video.uuid) } +function getHlsResolutionPlaylistFilename (videoFilename: string) { + // Video file name already contain resolution + return removeFragmentedMP4Ext(videoFilename) + '.m3u8' +} + +function generateHLSMasterPlaylistFilename (isLive = false) { + if (isLive) return 'master.m3u8' + + return buildUUID() + '-master.m3u8' +} + +function generateHlsSha256SegmentsFilename (isLive = false) { + if (isLive) return 'segments-sha256.json' + + return buildUUID() + '-segments-sha256.json' +} + // ################## Torrents ################## function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { @@ -91,6 +105,9 @@ export { getTorrentFilePath, getHLSDirectory, + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + getHlsResolutionPlaylistFilename, getLocalVideoFileMetadataUrl, diff --git a/server/lib/video.ts b/server/lib/video.ts index daf998704..61fee4949 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database' import { TagModel } from '@server/models/video/tag' import { VideoModel } from '@server/models/video/video' import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' +import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models' import { federateVideoIfNeeded } from './activitypub/videos' import { JobQueue } from './job-queue/job-queue' @@ -105,7 +105,7 @@ async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) { } } -async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) { +async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) { let dataInput: VideoTranscodingPayload if (videoFile.isAudio()) { diff --git a/server/models/actor/actor-follow.ts b/server/models/actor/actor-follow.ts index 83c00a22d..3080e02a6 100644 --- a/server/models/actor/actor-follow.ts +++ b/server/models/actor/actor-follow.ts @@ -19,8 +19,8 @@ import { UpdatedAt } from 'sequelize-typescript' import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' +import { doesExist } from '@server/helpers/database-utils' import { getServerActor } from '@server/models/application/application' -import { VideoModel } from '@server/models/video/video' import { MActorFollowActorsDefault, MActorFollowActorsDefaultSubscription, @@ -166,14 +166,8 @@ export class ActorFollowModel extends Model results.length === 1) + return doesExist(query, { actorId, followerActorId }) } static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise { diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index ccda023e0..d645be248 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -160,8 +160,8 @@ export class VideoRedundancyModel extends Model logger.error('Cannot delete %s files.', logIdentifier, { err })) + videoFile.Video.removeFileAndTorrent(videoFile, true) + .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) } if (instance.videoStreamingPlaylistId) { diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 6b1e59063..3310b3b46 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -182,8 +182,8 @@ function streamingPlaylistsModelToFormattedJSON ( return { id: playlist.id, type: playlist.type, - playlistUrl: playlist.playlistUrl, - segmentsSha256Url: playlist.segmentsSha256Url, + playlistUrl: playlist.getMasterPlaylistUrl(video), + segmentsSha256Url: playlist.getSha256SegmentsUrl(video), redundancies, files } @@ -331,7 +331,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { type: 'Link', name: 'sha256', mediaType: 'application/json' as 'application/json', - href: playlist.segmentsSha256Url + href: playlist.getSha256SegmentsUrl(video) }) addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || []) @@ -339,7 +339,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { url.push({ type: 'Link', mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', - href: playlist.playlistUrl, + href: playlist.getMasterPlaylistUrl(video), tag }) } diff --git a/server/models/video/sql/shared/video-tables.ts b/server/models/video/sql/shared/video-tables.ts index abdd22188..742d19099 100644 --- a/server/models/video/sql/shared/video-tables.ts +++ b/server/models/video/sql/shared/video-tables.ts @@ -92,12 +92,13 @@ export class VideoTables { } getStreamingPlaylistAttributes () { - let playlistKeys = [ 'id', 'playlistUrl', 'type' ] + let playlistKeys = [ 'id', 'playlistUrl', 'playlistFilename', 'type' ] if (this.mode === 'get') { playlistKeys = playlistKeys.concat([ 'p2pMediaLoaderInfohashes', 'p2pMediaLoaderPeerVersion', + 'segmentsSha256Filename', 'segmentsSha256Url', 'videoId', 'createdAt', diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 22cf63804..797a85a4e 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -1,7 +1,7 @@ import { remove } from 'fs-extra' import * as memoizee from 'memoizee' import { join } from 'path' -import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' +import { FindOptions, Op, Transaction } from 'sequelize' import { AllowNull, BelongsTo, @@ -21,6 +21,7 @@ import { import { Where } from 'sequelize/types/lib/utils' import validator from 'validator' import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' +import { doesExist } from '@server/helpers/database-utils' import { logger } from '@server/helpers/logger' import { extractVideo } from '@server/helpers/video' import { getTorrentFilePath } from '@server/lib/video-paths' @@ -250,14 +251,8 @@ export class VideoFileModel extends Model static doesInfohashExist (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - bind: { infoHash }, - raw: true - } - return VideoModel.sequelize.query(query, options) - .then(results => results.length === 1) + return doesExist(query, { infoHash }) } static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { @@ -266,6 +261,33 @@ export class VideoFileModel extends Model return !!videoFile } + static async doesOwnedTorrentFileExist (filename: string) { + const query = 'SELECT 1 FROM "videoFile" ' + + 'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' + + 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' + + 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' + + 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1' + + return doesExist(query, { filename }) + } + + static async doesOwnedWebTorrentVideoFileExist (filename: string) { + const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + + 'WHERE "filename" = $filename LIMIT 1' + + return doesExist(query, { filename }) + } + + static loadByFilename (filename: string) { + const query = { + where: { + filename + } + } + + return VideoFileModel.findOne(query) + } + static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { const query = { where: { @@ -443,10 +465,9 @@ export class VideoFileModel extends Model } getFileDownloadUrl (video: MVideoWithHost) { - const basePath = this.isHLS() - ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS - : STATIC_DOWNLOAD_PATHS.VIDEOS - const path = join(basePath, this.filename) + const path = this.isHLS() + ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) + : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`) if (video.isOwned()) return WEBSERVER.URL + path diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index d627e8c9d..b15d20cf9 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -1,19 +1,27 @@ import * as memoizee from 'memoizee' import { join } from 'path' -import { Op, QueryTypes } from 'sequelize' +import { Op } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { doesExist } from '@server/helpers/database-utils' import { VideoFileModel } from '@server/models/video/video-file' -import { MStreamingPlaylist } from '@server/types/models' +import { MStreamingPlaylist, MVideo } from '@server/types/models' +import { AttributesOnly } from '@shared/core-utils' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { sha1 } from '../../helpers/core-utils' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { isArrayOf } from '../../helpers/custom-validators/misc' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants' +import { + CONSTRAINTS_FIELDS, + MEMOIZE_LENGTH, + MEMOIZE_TTL, + P2P_MEDIA_LOADER_PEER_VERSION, + STATIC_PATHS, + WEBSERVER +} from '../../initializers/constants' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { throwIfNotValid } from '../utils' import { VideoModel } from './video' -import { AttributesOnly } from '@shared/core-utils' @Table({ tableName: 'videoStreamingPlaylist', @@ -43,7 +51,11 @@ export class VideoStreamingPlaylistModel extends Model throwIfNotValid(value, isActivityPubUrlValid, 'playlist url')) + @Column + playlistFilename: string + + @AllowNull(true) + @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true)) @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) playlistUrl: string @@ -57,7 +69,11 @@ export class VideoStreamingPlaylistModel extends Model throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url')) + @Column + segmentsSha256Filename: string + + @AllowNull(true) + @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true)) @Column segmentsSha256Url: string @@ -98,14 +114,8 @@ export class VideoStreamingPlaylistModel extends Model(query, options) - .then(results => results.length === 1) + return doesExist(query, { infoHash }) } static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) { @@ -125,7 +135,13 @@ export class VideoStreamingPlaylistModel extends Model { const options = { where: { type: VideoStreamingPlaylistType.HLS, @@ -155,30 +171,29 @@ export class VideoStreamingPlaylistModel extends Model>> { // Remove physical files and torrents instance.VideoFiles.forEach(file => { - tasks.push(instance.removeFile(file)) - tasks.push(file.removeTorrent()) + tasks.push(instance.removeFileAndTorrent(file)) }) // Remove playlists file @@ -1670,10 +1669,13 @@ export class VideoModel extends Model>> { .concat(toAdd) } - removeFile (videoFile: MVideoFile, isRedundancy = false) { + removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) { const filePath = getVideoFilePath(this, videoFile, isRedundancy) - return remove(filePath) - .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) + + const promises: Promise[] = [ remove(filePath) ] + if (!isRedundancy) promises.push(videoFile.removeTorrent()) + + return Promise.all(promises) } async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts index 20346113d..4acde3cc5 100644 --- a/server/tests/api/live/live-constraints.ts +++ b/server/tests/api/live/live-constraints.ts @@ -4,7 +4,7 @@ import 'mocha' import * as chai from 'chai' import { VideoPrivacy } from '@shared/models' import { - checkLiveCleanup, + checkLiveCleanupAfterSave, cleanupTests, ConfigCommand, createMultipleServers, @@ -43,7 +43,7 @@ describe('Test live constraints', function () { expect(video.duration).to.be.greaterThan(0) } - await checkLiveCleanup(servers[0], videoId, resolutions) + await checkLiveCleanupAfterSave(servers[0], videoId, resolutions) } async function waitUntilLivePublishedOnAllServers (videoId: string) { diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index bd15396ec..8f1fb78a5 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts @@ -4,7 +4,7 @@ import 'mocha' import * as chai from 'chai' import { FfmpegCommand } from 'fluent-ffmpeg' import { - checkLiveCleanup, + checkLiveCleanupAfterSave, cleanupTests, ConfigCommand, createMultipleServers, @@ -150,7 +150,7 @@ describe('Save replay setting', function () { await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) // No resolutions saved since we did not save replay - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) it('Should correctly terminate the stream on blacklist and delete the live', async function () { @@ -179,7 +179,7 @@ describe('Save replay setting', function () { await wait(5000) await waitJobs(servers) - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) it('Should correctly terminate the stream on delete and delete the video', async function () { @@ -203,7 +203,7 @@ describe('Save replay setting', function () { await waitJobs(servers) await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) }) @@ -259,7 +259,7 @@ describe('Save replay setting', function () { }) it('Should have cleaned up the live files', async function () { - await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ]) }) it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { @@ -287,7 +287,7 @@ describe('Save replay setting', function () { await wait(5000) await waitJobs(servers) - await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ]) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ]) }) it('Should correctly terminate the stream on delete and delete the video', async function () { @@ -310,7 +310,7 @@ describe('Save replay setting', function () { await waitJobs(servers) await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup(servers[0], liveVideoUUID, []) + await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, []) }) }) diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index 4676a840a..d555cff19 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts @@ -2,10 +2,10 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' +import { basename, join } from 'path' import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' import { - checkLiveCleanup, + checkLiveCleanupAfterSave, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, cleanupTests, @@ -506,6 +506,10 @@ describe('Test live', function () { await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200) await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200) + // We should have generated random filenames + expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') + expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json') + expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length) for (const resolution of resolutions) { @@ -520,7 +524,9 @@ describe('Test live', function () { expect(file.fps).to.be.approximately(30, 2) } - const filename = `${video.uuid}-${resolution}-fragmented.mp4` + const filename = basename(file.fileUrl) + expect(filename).to.not.contain(video.uuid) + const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename)) const probe = await ffprobePromise(segmentPath) @@ -537,7 +543,7 @@ describe('Test live', function () { it('Should correctly have cleaned up the live files', async function () { this.timeout(30000) - await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ]) + await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ]) }) }) diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts index 16372b039..d0ca82b07 100644 --- a/server/tests/api/users/users-multiple-servers.ts +++ b/server/tests/api/users/users-multiple-servers.ts @@ -58,10 +58,10 @@ describe('Test users with multiple servers', function () { const { uuid } = await servers[0].videos.upload({ token: userAccessToken }) videoUUID = uuid + await waitJobs(servers) + await saveVideoInServers(servers, videoUUID) } - - await waitJobs(servers) }) it('Should be able to update my display name', async function () { diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts index c94d92cf2..857859fd3 100644 --- a/server/tests/api/videos/resumable-upload.ts +++ b/server/tests/api/videos/resumable-upload.ts @@ -170,8 +170,13 @@ describe('Test resumable upload', function () { const size = 1000 + // Content length check seems to have changed in v16 + const expectedStatus = process.version.startsWith('v16') + ? HttpStatusCode.CONFLICT_409 + : HttpStatusCode.BAD_REQUEST_400 + const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}` - await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size }) + await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size }) await checkFileSize(uploadId, 0) }) }) diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 921d7ce64..961f0e617 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -2,7 +2,8 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' +import { basename, join } from 'path' +import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' import { checkDirectoryIsEmpty, checkResolutionsInMasterPlaylist, @@ -19,8 +20,6 @@ import { } from '@shared/extra-utils' import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' -import { uuidRegex } from '@shared/core-utils' -import { basename } from 'path/posix' const expect = chai.expect @@ -78,11 +77,13 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h // Check resolution playlists { for (const resolution of resolutions) { + const file = hlsFiles.find(f => f.resolution.id === resolution) + const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' + const subPlaylist = await server.streamingPlaylists.get({ - url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8` + url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` }) - const file = hlsFiles.find(f => f.resolution.id === resolution) expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) expect(subPlaylist).to.contain(basename(file.fileUrl)) } diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts index 685b3b7b8..579b2e7d8 100644 --- a/server/tests/cli/optimize-old-videos.ts +++ b/server/tests/cli/optimize-old-videos.ts @@ -2,7 +2,6 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' import { cleanupTests, createMultipleServers, @@ -86,7 +85,7 @@ describe('Test optimize old videos', function () { expect(file.size).to.be.below(8000000) - const path = servers[0].servers.buildDirectory(join('videos', video.uuid + '-' + file.resolution.id + '.mp4')) + const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl) const bitrate = await getVideoFileBitrate(path) const fps = await getVideoFileFPS(path) const resolution = await getVideoFileResolution(path) diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 954a87833..2d4c02da7 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts @@ -36,7 +36,7 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst } } -async function assertCountAreOkay (servers: PeerTubeServer[]) { +async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) { for (const server of servers) { const videosCount = await countFiles(server, 'videos') expect(videosCount).to.equal(8) @@ -53,12 +53,21 @@ async function assertCountAreOkay (servers: PeerTubeServer[]) { const avatarsCount = await countFiles(server, 'avatars') expect(avatarsCount).to.equal(2) } + + // When we'll prune HLS directories too + // const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/') + // expect(hlsRootCount).to.equal(2) + + // const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID) + // expect(hlsCount).to.equal(10) } describe('Test prune storage scripts', function () { let servers: PeerTubeServer[] const badNames: { [directory: string]: string[] } = {} + let videoServer2UUID: string + before(async function () { this.timeout(120000) @@ -68,7 +77,9 @@ describe('Test prune storage scripts', function () { for (const server of servers) { await server.videos.upload({ attributes: { name: 'video 1' } }) - await server.videos.upload({ attributes: { name: 'video 2' } }) + + const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } }) + if (server.serverNumber === 2) videoServer2UUID = uuid await server.users.updateMyAvatar({ fixture: 'avatar.png' }) @@ -112,7 +123,7 @@ describe('Test prune storage scripts', function () { }) it('Should have the files on the disk', async function () { - await assertCountAreOkay(servers) + await assertCountAreOkay(servers, videoServer2UUID) }) it('Should create some dirty files', async function () { @@ -176,6 +187,28 @@ describe('Test prune storage scripts', function () { badNames['avatars'] = [ n1, n2 ] } + + // When we'll prune HLS directories too + // { + // const directory = join('streaming-playlists', 'hls') + // const base = servers[1].servers.buildDirectory(directory) + + // const n1 = buildUUID() + // await createFile(join(base, n1)) + // badNames[directory] = [ n1 ] + // } + + // { + // const directory = join('streaming-playlists', 'hls', videoServer2UUID) + // const base = servers[1].servers.buildDirectory(directory) + // const n1 = buildUUID() + '-240-fragmented-.mp4' + // const n2 = buildUUID() + '-master.m3u8' + + // await createFile(join(base, n1)) + // await createFile(join(base, n2)) + + // badNames[directory] = [ n1, n2 ] + // } } }) @@ -187,7 +220,7 @@ describe('Test prune storage scripts', function () { }) it('Should have removed files', async function () { - await assertCountAreOkay(servers) + await assertCountAreOkay(servers, videoServer2UUID) for (const directory of Object.keys(badNames)) { for (const name of badNames[directory]) { diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index fcbcb55ba..43fbaec30 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts @@ -108,21 +108,22 @@ describe('Test update host scripts', function () { for (const video of data) { const videoDetails = await server.videos.get({ id: video.id }) + const files = videoDetails.files.concat(videoDetails.streamingPlaylists[0].files) - expect(videoDetails.files).to.have.lengthOf(4) + expect(files).to.have.lengthOf(8) - for (const file of videoDetails.files) { + for (const file of files) { expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket') - expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F') + expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2F') - const torrent = await parseTorrentVideo(server, videoDetails.uuid, file.resolution.id) + const torrent = await parseTorrentVideo(server, file) const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket') expect(announceWS).to.not.be.undefined const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce') expect(announceHttp).to.not.be.undefined - expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed') + expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/') } } }) diff --git a/server/tests/plugins/plugin-transcoding.ts b/server/tests/plugins/plugin-transcoding.ts index c14c34c7e..93637e3ce 100644 --- a/server/tests/plugins/plugin-transcoding.ts +++ b/server/tests/plugins/plugin-transcoding.ts @@ -2,7 +2,6 @@ import 'mocha' import { expect } from 'chai' -import { join } from 'path' import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' import { cleanupTests, @@ -247,7 +246,9 @@ describe('Test transcoding plugins', function () { const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid await waitJobs([ server ]) - const path = server.servers.buildDirectory(join('videos', videoUUID + '-240.mp4')) + const video = await server.videos.get({ id: videoUUID }) + + const path = server.servers.buildWebTorrentFilePath(video.files[0].fileUrl) const audioProbe = await getAudioStream(path) expect(audioProbe.audioStream.codec_name).to.equal('opus') diff --git a/server/types/models/video/video-streaming-playlist.ts b/server/types/models/video/video-streaming-playlist.ts index 8b3ef51fc..1e4dccb8e 100644 --- a/server/types/models/video/video-streaming-playlist.ts +++ b/server/types/models/video/video-streaming-playlist.ts @@ -39,5 +39,5 @@ export type MStreamingPlaylistRedundanciesOpt = PickWithOpt export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo { - return !!(value as MStreamingPlaylist).playlistUrl + return !!(value as MStreamingPlaylist).videoId } -- cgit v1.2.3