From 90a8bd305de4153ec21137a73ff482dcc2e3e19b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 16 Feb 2021 16:25:53 +0100 Subject: Dissociate video file names and video uuid --- server/lib/activitypub/videos.ts | 54 +++++++++++----- .../abstract-video-static-file-cache.ts | 2 +- server/lib/files-cache/videos-torrent-cache.ts | 54 ++++++++++++++++ server/lib/hls.ts | 4 +- server/lib/job-queue/handlers/video-file-import.ts | 14 ++-- server/lib/job-queue/handlers/video-import.ts | 8 ++- server/lib/job-queue/handlers/video-live-ending.ts | 2 +- server/lib/job-queue/handlers/video-transcoding.ts | 2 +- server/lib/live-manager.ts | 6 +- .../lib/schedulers/videos-redundancy-scheduler.ts | 12 ++-- server/lib/video-paths.ts | 75 ++++++++++++++++------ server/lib/video-transcoding.ts | 33 ++++++---- 12 files changed, 194 insertions(+), 72 deletions(-) create mode 100644 server/lib/files-cache/videos-torrent-cache.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 66981f43f..a5f6537eb 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -1,7 +1,7 @@ import * as Bluebird from 'bluebird' import { maxBy, minBy } from 'lodash' import * as magnetUtil from 'magnet-uri' -import { join } from 'path' +import { basename, join } from 'path' import * as request from 'request' import * as sequelize from 'sequelize' import { VideoLiveModel } from '@server/models/video/video-live' @@ -30,11 +30,11 @@ import { doRequest } from '../../helpers/requests' import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' import { ACTIVITY_PUB, + LAZY_STATIC_PATHS, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, REMOTE_SCHEME, - STATIC_PATHS, THUMBNAILS_SIZE } from '../../initializers/constants' import { sequelizeTypescript } from '../../initializers/database' @@ -51,6 +51,8 @@ import { MChannelDefault, MChannelId, MStreamingPlaylist, + MStreamingPlaylistFilesVideo, + MStreamingPlaylistVideo, MVideo, MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, @@ -61,7 +63,8 @@ import { MVideoFullLight, MVideoId, MVideoImmutable, - MVideoThumbnail + MVideoThumbnail, + MVideoWithHost } from '../../types/models' import { MThumbnail } from '../../types/models/video/thumbnail' import { FilteredModelAttributes } from '../../types/sequelize' @@ -72,6 +75,7 @@ import { PeerTubeSocket } from '../peertube-socket' import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail' import { setVideoTags } from '../video' import { autoBlacklistVideoIfNeeded } from '../video-blacklist' +import { generateTorrentFileName } from '../video-paths' import { getOrCreateActorAndServerAndModel } from './actor' import { crawlCollectionPage } from './crawl' import { sendCreateVideo, sendUpdateVideo } from './send' @@ -405,7 +409,8 @@ async function updateVideoFromAP (options: { for (const playlistAttributes of streamingPlaylistAttributes) { const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t }) - .then(([ streamingPlaylist ]) => streamingPlaylist) + .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo) + streamingPlaylistModel.Video = videoUpdated const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject) .map(a => new VideoFileModel(a)) @@ -637,13 +642,14 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi videoCreated.VideoStreamingPlaylists = [] for (const playlistAttributes of streamingPlaylistsAttributes) { - const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) + const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo + playlist.Video = videoCreated - const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject) + const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject) const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t })) - playlistModel.VideoFiles = await Promise.all(videoFilePromises) + playlist.VideoFiles = await Promise.all(videoFilePromises) - videoCreated.VideoStreamingPlaylists.push(playlistModel) + videoCreated.VideoStreamingPlaylists.push(playlist) } // Process tags @@ -766,7 +772,7 @@ function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObjec } function videoFileActivityUrlToDBAttributes ( - videoOrPlaylist: MVideo | MStreamingPlaylist, + videoOrPlaylist: MVideo | MStreamingPlaylistVideo, urls: (ActivityTagObject | ActivityUrlObject)[] ) { const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] @@ -786,6 +792,10 @@ function videoFileActivityUrlToDBAttributes ( throw new Error('Cannot parse magnet URI ' + magnet.href) } + const torrentUrl = Array.isArray(parsed.xs) + ? parsed.xs[0] + : parsed.xs + // Fetch associated metadata url, if any const metadata = urls.filter(isAPVideoFileMetadataObject) .find(u => { @@ -794,18 +804,30 @@ function videoFileActivityUrlToDBAttributes ( u.rel.includes(fileUrl.mediaType) }) - const mediaType = fileUrl.mediaType + const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) + const resolution = fileUrl.height + const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id + const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null + const attribute = { - extname: getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, mediaType), + extname, infoHash: parsed.infoHash, - resolution: fileUrl.height, + resolution, size: fileUrl.size, fps: fileUrl.fps || -1, metadataUrl: metadata?.href, + // Use the name of the remote file because we don't proxify video file requests + filename: basename(fileUrl.href), + fileUrl: fileUrl.href, + + torrentUrl, + // Use our own torrent name since we proxify torrent requests + torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), + // This is a video file owned by a video or by a streaming playlist - videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id, - videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null + videoId, + videoStreamingPlaylistId } attributes.push(attribute) @@ -862,8 +884,8 @@ function getPreviewFromIcons (videoObject: VideoObject) { return maxBy(validIcons, 'width') } -function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoAccountLight) { +function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoWithHost) { return previewIcon ? previewIcon.url - : buildRemoteVideoBaseUrl(video, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) + : buildRemoteVideoBaseUrl(video, join(LAZY_STATIC_PATHS.PREVIEWS, video.generatePreviewName())) } diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts index c06355446..af66689a0 100644 --- a/server/lib/files-cache/abstract-video-static-file-cache.ts +++ b/server/lib/files-cache/abstract-video-static-file-cache.ts @@ -2,7 +2,7 @@ import { remove } from 'fs-extra' import { logger } from '../../helpers/logger' import * as memoizee from 'memoizee' -type GetFilePathResult = { isOwned: boolean, path: string } | undefined +type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined export abstract class AbstractVideoStaticFileCache { diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/videos-torrent-cache.ts new file mode 100644 index 000000000..ca0e1770d --- /dev/null +++ b/server/lib/files-cache/videos-torrent-cache.ts @@ -0,0 +1,54 @@ +import { join } from 'path' +import { doRequestAndSaveToFile } from '@server/helpers/requests' +import { VideoFileModel } from '@server/models/video/video-file' +import { CONFIG } from '../../initializers/config' +import { FILES_CACHE } from '../../initializers/constants' +import { VideoModel } from '../../models/video/video' +import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' + +class VideosTorrentCache extends AbstractVideoStaticFileCache { + + private static instance: VideosTorrentCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(filename) + if (!file) return undefined + + if (file.getVideo().isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.TORRENTS_DIR, file.torrentFilename) } + + return this.loadRemoteFile(filename) + } + + // Key is the torrent filename + protected async loadRemoteFile (key: string) { + const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(key) + if (!file) return undefined + + if (file.getVideo().isOwned()) throw new Error('Cannot load remote file of owned video.') + + // Used to fetch the path + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.getVideo().id) + if (!video) return undefined + + const remoteUrl = file.getRemoteTorrentUrl(video) + const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) + + await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) + + const downloadName = `${video.name}-${file.resolution}p.torrent` + + return { isOwned: false, path: destPath, downloadName } + } +} + +export { + VideosTorrentCache +} diff --git a/server/lib/hls.ts b/server/lib/hls.ts index ef489097a..04187668c 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { getVideoFilename, getVideoFilePath } from './video-paths' +import { getVideoFilePath } from './video-paths' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -93,7 +93,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) { } await close(fd) - const videoFilename = getVideoFilename(hlsPlaylist, file) + const videoFilename = file.filename json[videoFilename] = rangeHashes } diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index cd95aa075..86c9b5c29 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -2,9 +2,9 @@ import * as Bull from 'bull' import { copy, stat } from 'fs-extra' import { extname } from 'path' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { getVideoFilePath } from '@server/lib/video-paths' +import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { UserModel } from '@server/models/account/user' -import { MVideoFile, MVideoWithFile } from '@server/types/models' +import { MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoFileImportPayload } from '@shared/models' import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' import { logger } from '../../../helpers/logger' @@ -50,14 +50,16 @@ export { // --------------------------------------------------------------------------- -async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { +async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { const { videoFileResolution } = await getVideoFileResolution(inputFilePath) const { size } = await stat(inputFilePath) const fps = await getVideoFileFPS(inputFilePath) + const fileExt = extname(inputFilePath) let updatedVideoFile = new VideoFileModel({ resolution: videoFileResolution, - extname: extname(inputFilePath), + extname: fileExt, + filename: generateVideoFilename(video, false, videoFileResolution, fileExt), size, fps, videoId: video.id @@ -68,7 +70,7 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { if (currentVideoFile) { // Remove old file and old torrent await video.removeFile(currentVideoFile) - await video.removeTorrent(currentVideoFile) + await currentVideoFile.removeTorrent() // Remove the old video file from the array video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) @@ -83,7 +85,7 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { const outputPath = getVideoFilePath(video, updatedVideoFile) await copy(inputFilePath, outputPath) - await createTorrentAndSetInfoHash(video, updatedVideoFile) + await createTorrentAndSetInfoHash(video, video, updatedVideoFile) await updatedVideoFile.save() diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 0d00c1b9d..8fa024105 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -6,7 +6,7 @@ import { isPostImportVideoAccepted } from '@server/lib/moderation' import { Hooks } from '@server/lib/plugins/hooks' import { isAbleToUploadVideo } from '@server/lib/user' import { addOptimizeOrMergeAudioJob } from '@server/lib/video' -import { getVideoFilePath } from '@server/lib/video-paths' +import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { ThumbnailModel } from '@server/models/video/thumbnail' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' import { @@ -116,10 +116,12 @@ async function processFile (downloader: () => Promise, videoImport: MVid const duration = await getDurationFromVideoFile(tempVideoPath) // Prepare video file object for creation in database + const fileExt = extname(tempVideoPath) const videoFileData = { - extname: extname(tempVideoPath), + extname: fileExt, resolution: videoFileResolution, size: stats.size, + filename: generateVideoFilename(videoImport.Video, false, videoFileResolution, fileExt), fps, videoId: videoImport.videoId } @@ -183,7 +185,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid } // Create torrent - await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) + await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoImportWithFiles.Video, videoFile) const videoFileSave = videoFile.toJSON() diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 6d50635bb..d57202ca5 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -85,7 +85,7 @@ async function saveLive (video: MVideo, live: MVideoLive) { await video.save() // Remove old HLS playlist video files - const videoWithFiles = await VideoModel.loadWithFiles(video.id) + const videoWithFiles = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id) const hlsPlaylist = videoWithFiles.getHLSPlaylist() await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index e248b645e..8573d4d12 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -128,7 +128,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay // Remove webtorrent files if not enabled for (const file of video.VideoFiles) { await video.removeFile(file) - await video.removeTorrent(file) + await file.removeTorrent() await file.destroy() } diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts index 9f17b8820..b549c189f 100644 --- a/server/lib/live-manager.ts +++ b/server/lib/live-manager.ts @@ -16,7 +16,7 @@ import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylist, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' +import { MStreamingPlaylist, MStreamingPlaylistVideo, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' import { VideoState, VideoStreamingPlaylistType } from '@shared/models' import { federateVideoIfNeeded } from './activitypub/videos' import { buildSha256Segment } from './hls' @@ -277,7 +277,7 @@ class LiveManager { return this.runMuxing({ sessionId, videoLive, - playlist: videoStreamingPlaylist, + playlist: Object.assign(videoStreamingPlaylist, { Video: video }), rtmpUrl, fps, allResolutions @@ -287,7 +287,7 @@ class LiveManager { private async runMuxing (options: { sessionId: string videoLive: MVideoLiveVideo - playlist: MStreamingPlaylist + playlist: MStreamingPlaylistVideo rtmpUrl: string fps: number allResolutions: number[] diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 93e76626c..60008e695 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -18,14 +18,14 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { CONFIG } from '../../initializers/config' -import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' +import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' import { downloadPlaylistSegments } from '../hls' import { removeVideoRedundancy } from '../redundancy' -import { getVideoFilename } from '../video-paths' +import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' import { AbstractScheduler } from './abstract-scheduler' type CandidateToDuplicate = { @@ -222,17 +222,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) + const magnetUri = generateMagnetUri(video, video, file, baseUrlHttp, baseUrlWs) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, getVideoFilename(video, file)) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ expiresOn, url: getLocalVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), + fileUrl: generateWebTorrentRedundancyUrl(file), strategy, videoFileId: file.id, actorId: serverActor.id @@ -271,7 +271,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ expiresOn, url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), - fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), + fileUrl: generateHLSRedundancyUrl(video, playlistArg), strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index 53fc8e81d..0385e89cc 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts @@ -1,19 +1,23 @@ -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' import { join } from 'path' -import { CONFIG } from '@server/initializers/config' -import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' import { extractVideo } from '@server/helpers/video' +import { CONFIG } from '@server/initializers/config' +import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' +import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' // ################## Video file name ################## -function getVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { +function generateVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, isHls: boolean, resolution: number, extname: string) { const video = extractVideo(videoOrPlaylist) - if (videoFile.isHLS()) { - return generateVideoStreamingPlaylistName(video.uuid, videoFile.resolution) + // FIXME: use a generated uuid instead, that will break compatibility with PeerTube < 3.2 + // const uuid = uuidv4() + const uuid = video.uuid + + if (isHls) { + return generateVideoStreamingPlaylistName(uuid, resolution) } - return generateWebTorrentVideoName(video.uuid, videoFile.resolution, videoFile.extname) + return generateWebTorrentVideoName(uuid, resolution, extname) } function generateVideoStreamingPlaylistName (uuid: string, resolution: number) { @@ -28,36 +32,64 @@ function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, vi if (videoFile.isHLS()) { const video = extractVideo(videoOrPlaylist) - return join(getHLSDirectory(video), getVideoFilename(videoOrPlaylist, videoFile)) + return join(getHLSDirectory(video), videoFile.filename) } - const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR - return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile)) + const baseDir = isRedundancy + ? CONFIG.STORAGE.REDUNDANCY_DIR + : CONFIG.STORAGE.VIDEOS_DIR + + return join(baseDir, videoFile.filename) +} + +// ################## Redundancy ################## + +function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { + // Base URL used by our HLS player + return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid +} + +function generateWebTorrentRedundancyUrl (file: MVideoFile) { + return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename } // ################## Streaming playlist ################## function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { - const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY + const baseDir = isRedundancy + ? HLS_REDUNDANCY_DIRECTORY + : HLS_STREAMING_PLAYLIST_DIRECTORY return join(baseDir, video.uuid) } // ################## Torrents ################## -function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { +function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { const video = extractVideo(videoOrPlaylist) const extension = '.torrent' + // FIXME: use a generated uuid instead, that will break compatibility with PeerTube < 3.2 + // const uuid = uuidv4() + const uuid = video.uuid + if (isStreamingPlaylist(videoOrPlaylist)) { - return `${video.uuid}-${videoFile.resolution}-${videoOrPlaylist.getStringType()}${extension}` + return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` } - return video.uuid + '-' + videoFile.resolution + extension + return uuid + '-' + resolution + extension +} + +function getTorrentFilePath (videoFile: MVideoFile) { + return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) } -function getTorrentFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - return join(CONFIG.STORAGE.TORRENTS_DIR, getTorrentFileName(videoOrPlaylist, videoFile)) +// ################## Meta data ################## + +function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { + const path = '/api/v1/videos/' + + return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id } // --------------------------------------------------------------------------- @@ -65,11 +97,16 @@ function getTorrentFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, export { generateVideoStreamingPlaylistName, generateWebTorrentVideoName, - getVideoFilename, + generateVideoFilename, getVideoFilePath, - getTorrentFileName, + generateTorrentFileName, getTorrentFilePath, - getHLSDirectory + getHLSDirectory, + + getLocalVideoFileMetadataUrl, + + generateWebTorrentRedundancyUrl, + generateHLSRedundancyUrl } diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index a58c9dd20..b366e2e44 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -2,7 +2,7 @@ import { Job } from 'bull' import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' import { basename, extname as extnameUtil, join } from 'path' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoWithAllFiles, MVideoWithFile } from '@server/types/models' +import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoResolution } from '../../shared/models/videos' import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' import { transcode, TranscodeOptions, TranscodeOptionsType } from '../helpers/ffmpeg-utils' @@ -13,7 +13,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSER import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { updateMasterHLSPlaylist, updateSha256VODSegments } from './hls' -import { generateVideoStreamingPlaylistName, getVideoFilename, getVideoFilePath } from './video-paths' +import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from './video-paths' import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' /** @@ -24,7 +24,7 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' */ // Optimize the original video file and replace it. The resolution is not changed. -async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: MVideoFile, job?: Job) { +async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' @@ -55,8 +55,9 @@ async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: try { await remove(videoInputPath) - // Important to do this before getVideoFilename() to take in account the new file extension + // Important to do this before getVideoFilename() to take in account the new filename inputVideoFile.extname = newExtname + inputVideoFile.filename = generateVideoFilename(video, false, inputVideoFile.resolution, newExtname) const videoOutputPath = getVideoFilePath(video, inputVideoFile) @@ -72,7 +73,7 @@ async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: } // Transcode the original video file to a lower resolution. -async function transcodeNewWebTorrentResolution (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean, job: Job) { +async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const extname = '.mp4' @@ -82,11 +83,13 @@ async function transcodeNewWebTorrentResolution (video: MVideoWithFile, resoluti const newVideoFile = new VideoFileModel({ resolution, extname, + filename: generateVideoFilename(video, false, resolution, extname), size: 0, videoId: video.id }) + const videoOutputPath = getVideoFilePath(video, newVideoFile) - const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile)) + const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) const transcodeOptions = resolution === VideoResolution.H_NOVIDEO ? { @@ -122,7 +125,7 @@ async function transcodeNewWebTorrentResolution (video: MVideoWithFile, resoluti } // Merge an image with an audio file to create a video -async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution, job: Job) { +async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' @@ -175,7 +178,7 @@ async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: Video // Concat TS segments from a live video to a fragmented mp4 HLS playlist async function generateHlsPlaylistResolutionFromTS (options: { - video: MVideoWithFile + video: MVideoFullLight concatenatedTsFilePath: string resolution: VideoResolution isPortraitMode: boolean @@ -193,7 +196,7 @@ async function generateHlsPlaylistResolutionFromTS (options: { // Generate an HLS playlist from an input file, and update the master playlist function generateHlsPlaylistResolution (options: { - video: MVideoWithFile + video: MVideoFullLight videoInputPath: string resolution: VideoResolution copyCodecs: boolean @@ -235,7 +238,7 @@ export { // --------------------------------------------------------------------------- async function onWebTorrentVideoFileTranscoding ( - video: MVideoWithFile, + video: MVideoFullLight, videoFile: MVideoFile, transcodingPath: string, outputPath: string @@ -250,7 +253,7 @@ async function onWebTorrentVideoFileTranscoding ( videoFile.fps = fps videoFile.metadata = metadata - await createTorrentAndSetInfoHash(video, videoFile) + await createTorrentAndSetInfoHash(video, video, videoFile) await VideoFileModel.customUpsert(videoFile, 'video', undefined) video.VideoFiles = await video.$get('VideoFiles') @@ -260,7 +263,7 @@ async function onWebTorrentVideoFileTranscoding ( async function generateHlsPlaylistCommon (options: { type: 'hls' | 'hls-from-ts' - video: MVideoWithFile + video: MVideoFullLight inputPath: string resolution: VideoResolution copyCodecs?: boolean @@ -318,10 +321,12 @@ async function generateHlsPlaylistCommon (options: { videoStreamingPlaylist.Video = video // Build the new playlist file + const extname = extnameUtil(videoFilename) const newVideoFile = new VideoFileModel({ resolution, - extname: extnameUtil(videoFilename), + extname, size: 0, + filename: generateVideoFilename(video, true, resolution, extname), fps: -1, videoStreamingPlaylistId: videoStreamingPlaylist.id }) @@ -344,7 +349,7 @@ async function generateHlsPlaylistCommon (options: { newVideoFile.fps = await getVideoFileFPS(videoFilePath) newVideoFile.metadata = await getMetadataFromFile(videoFilePath) - await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) + await createTorrentAndSetInfoHash(videoStreamingPlaylist, video, newVideoFile) await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles') -- cgit v1.2.3