From 90a8bd305de4153ec21137a73ff482dcc2e3e19b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 16 Feb 2021 16:25:53 +0100 Subject: [PATCH] Dissociate video file names and video uuid --- config/default.yaml | 2 + config/production.yaml.example | 2 + scripts/optimize-old-videos.ts | 6 +- scripts/update-host.ts | 8 +- server.ts | 6 +- server/controllers/api/videos/index.ts | 7 +- server/controllers/download.ts | 78 ++++++++ server/controllers/index.ts | 1 + server/controllers/lazy-static.ts | 23 ++- server/controllers/static.ts | 88 +-------- server/helpers/activitypub.ts | 12 +- server/helpers/webtorrent.ts | 57 +++--- server/initializers/checker-before-init.ts | 2 +- server/initializers/config.ts | 3 + server/initializers/constants.ts | 13 +- server/lib/activitypub/videos.ts | 54 ++++-- .../abstract-video-static-file-cache.ts | 2 +- .../lib/files-cache/videos-torrent-cache.ts | 54 ++++++ server/lib/hls.ts | 4 +- .../job-queue/handlers/video-file-import.ts | 14 +- server/lib/job-queue/handlers/video-import.ts | 8 +- .../job-queue/handlers/video-live-ending.ts | 2 +- .../job-queue/handlers/video-transcoding.ts | 2 +- server/lib/live-manager.ts | 6 +- .../schedulers/videos-redundancy-scheduler.ts | 12 +- server/lib/video-paths.ts | 75 ++++++-- server/lib/video-transcoding.ts | 33 ++-- server/models/video/thumbnail.ts | 4 +- server/models/video/video-caption.ts | 6 +- server/models/video/video-file.ts | 181 +++++++++++++++--- server/models/video/video-format-utils.ts | 64 ++++--- server/models/video/video-query-builder.ts | 8 + .../models/video/video-streaming-playlist.ts | 58 ++---- server/models/video/video.ts | 60 ++---- server/tests/api/videos/video-hls.ts | 2 +- .../tests/cli/create-import-video-file-job.ts | 16 +- server/types/models/video/video-channels.ts | 5 + server/types/models/video/video.ts | 23 ++- shared/extra-utils/videos/videos.ts | 22 ++- shared/models/videos/video-file.model.ts | 8 +- 40 files changed, 637 insertions(+), 394 deletions(-) create mode 100644 server/controllers/download.ts create mode 100644 server/lib/files-cache/videos-torrent-cache.ts diff --git a/config/default.yaml b/config/default.yaml index 3bbb3e5c4..2d8afe1c3 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -197,6 +197,8 @@ cache: size: 500 # Max number of previews you want to cache captions: size: 500 # Max number of video captions/subtitles you want to cache + torrents: + size: 500 # Max number of video torrents you want to cache admin: # Used to generate the root user at first startup diff --git a/config/production.yaml.example b/config/production.yaml.example index d75e30276..2794c543c 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -208,6 +208,8 @@ cache: size: 500 # Max number of previews you want to cache captions: size: 500 # Max number of video captions/subtitles you want to cache + torrents: + size: 500 # Max number of video torrents you want to cache admin: # Used to generate the root user at first startup diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index d5696de67..8e2e7fcf4 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts @@ -34,7 +34,9 @@ async function run () { const localVideos = await VideoModel.listLocal() - for (const video of localVideos) { + for (const localVideo of localVideos) { + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id) + currentVideoId = video.id for (const file of video.VideoFiles) { @@ -70,7 +72,7 @@ async function run () { console.log('Failed to optimize %s, restoring original', basename(currentFile)) await move(backupFile, currentFile, { overwrite: true }) - await createTorrentAndSetInfoHash(video, file) + await createTorrentAndSetInfoHash(video, video, file) await file.save() } } diff --git a/scripts/update-host.ts b/scripts/update-host.ts index b030b21c3..d0a1b03cc 100755 --- a/scripts/update-host.ts +++ b/scripts/update-host.ts @@ -116,8 +116,10 @@ async function run () { console.log('Updating video and torrent files.') - const videos = await VideoModel.listLocal() - for (const video of videos) { + const localVideos = await VideoModel.listLocal() + for (const localVideo of localVideos) { + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id) + console.log('Updating video ' + video.uuid) video.url = getLocalVideoActivityPubUrl(video) @@ -125,7 +127,7 @@ async function run () { for (const file of video.VideoFiles) { console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) - await createTorrentAndSetInfoHash(video, file) + await createTorrentAndSetInfoHash(video, video, file) } for (const playlist of video.VideoStreamingPlaylists) { diff --git a/server.ts b/server.ts index 66dcb3c40..00cd87e20 100644 --- a/server.ts +++ b/server.ts @@ -103,7 +103,8 @@ import { webfingerRouter, trackerRouter, createWebsocketTrackerServer, - botsRouter + botsRouter, + downloadRouter } from './server/controllers' import { advertiseDoNotTrack } from './server/middlewares/dnt' import { Redis } from './server/lib/redis' @@ -123,6 +124,7 @@ import { Hooks } from './server/lib/plugins/hooks' import { PluginManager } from './server/lib/plugins/plugin-manager' import { LiveManager } from './server/lib/live-manager' import { HttpStatusCode } from './shared/core-utils/miscs/http-error-codes' +import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' // ----------- Command line ----------- @@ -202,6 +204,7 @@ app.use('/', botsRouter) // Static files app.use('/', staticRouter) +app.use('/', downloadRouter) app.use('/', lazyStaticRouter) // Client files, last valid routes! @@ -258,6 +261,7 @@ async function startApplication () { // Caches initializations VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE) VideosCaptionCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE) + VideosTorrentCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE) // Enable Schedulers ActorFollowScheduler.Instance.enable() diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 9504c40a4..dcd6194ae 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -7,7 +7,7 @@ import { changeVideoChannelShare } from '@server/lib/activitypub/share' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { LiveManager } from '@server/lib/live-manager' import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' -import { getVideoFilePath } from '@server/lib/video-paths' +import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { getServerActor } from '@server/models/application/application' import { MVideoFullLight } from '@server/types/models' import { VideoCreate, VideoState, VideoUpdate } from '../../../../shared' @@ -189,6 +189,7 @@ async function addVideo (req: express.Request, res: express.Response) { videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware const video = new VideoModel(videoData) as MVideoFullLight + video.VideoChannel = res.locals.videoChannel video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object const videoFile = new VideoFileModel({ @@ -205,6 +206,8 @@ async function addVideo (req: express.Request, res: express.Response) { videoFile.resolution = (await getVideoFileResolution(videoPhysicalFile.path)).videoFileResolution } + videoFile.filename = generateVideoFilename(video, false, videoFile.resolution, videoFile.extname) + // Move physical file const destination = getVideoFilePath(video, videoFile) await move(videoPhysicalFile.path, destination) @@ -219,7 +222,7 @@ async function addVideo (req: express.Request, res: express.Response) { }) // Create the torrent file - await createTorrentAndSetInfoHash(video, videoFile) + await createTorrentAndSetInfoHash(video, video, videoFile) const { videoCreated } = await sequelizeTypescript.transaction(async t => { const sequelizeOptions = { transaction: t } diff --git a/server/controllers/download.ts b/server/controllers/download.ts new file mode 100644 index 000000000..27caa1518 --- /dev/null +++ b/server/controllers/download.ts @@ -0,0 +1,78 @@ +import * as cors from 'cors' +import * as express from 'express' +import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' +import { getVideoFilePath } from '@server/lib/video-paths' +import { MVideoFile, MVideoFullLight } from '@server/types/models' +import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes' +import { VideoStreamingPlaylistType } from '@shared/models' +import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' +import { asyncMiddleware, videosDownloadValidator } from '../middlewares' + +const downloadRouter = express.Router() + +downloadRouter.use(cors()) + +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.TORRENTS + ':filename', + downloadTorrent +) + +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', + asyncMiddleware(videosDownloadValidator), + downloadVideoFile +) + +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', + asyncMiddleware(videosDownloadValidator), + downloadHLSVideoFile +) + +// --------------------------------------------------------------------------- + +export { + downloadRouter +} + +// --------------------------------------------------------------------------- + +async function downloadTorrent (req: express.Request, res: express.Response) { + const result = await VideosTorrentCache.Instance.getFilePath(req.params.filename) + if (!result) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + + return res.download(result.path, result.downloadName) +} + +function downloadVideoFile (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const videoFile = getVideoFile(req, video.VideoFiles) + if (!videoFile) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) +} + +function downloadHLSVideoFile (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const playlist = getHLSPlaylist(video) + if (!playlist) return res.status(HttpStatusCode.NOT_FOUND_404).end + + const videoFile = getVideoFile(req, playlist.VideoFiles) + if (!videoFile) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + const filename = `${video.name}-${videoFile.resolution}p-${playlist.getStringType()}${videoFile.extname}` + return res.download(getVideoFilePath(playlist, videoFile), filename) +} + +function getVideoFile (req: express.Request, files: MVideoFile[]) { + const resolution = parseInt(req.params.resolution, 10) + return files.find(f => f.resolution === resolution) +} + +function getHLSPlaylist (video: MVideoFullLight) { + const playlist = video.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + if (!playlist) return undefined + + return Object.assign(playlist, { Video: video }) +} diff --git a/server/controllers/index.ts b/server/controllers/index.ts index 5a199ae9c..fa27ecec2 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts @@ -1,6 +1,7 @@ export * from './activitypub' export * from './api' export * from './client' +export * from './download' export * from './feeds' export * from './services' export * from './static' diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts index 656dea223..c2f5c7b56 100644 --- a/server/controllers/lazy-static.ts +++ b/server/controllers/lazy-static.ts @@ -1,12 +1,13 @@ import * as cors from 'cors' import * as express from 'express' +import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache' +import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' +import { logger } from '../helpers/logger' import { LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' +import { avatarPathUnsafeCache, pushAvatarProcessInQueue } from '../lib/avatar' import { VideosCaptionCache, VideosPreviewCache } from '../lib/files-cache' import { asyncMiddleware } from '../middlewares' import { AvatarModel } from '../models/avatar/avatar' -import { logger } from '../helpers/logger' -import { avatarPathUnsafeCache, pushAvatarProcessInQueue } from '../lib/avatar' -import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' const lazyStaticRouter = express.Router() @@ -27,6 +28,11 @@ lazyStaticRouter.use( asyncMiddleware(getVideoCaption) ) +lazyStaticRouter.use( + LAZY_STATIC_PATHS.TORRENTS + ':filename', + asyncMiddleware(getTorrent) +) + // --------------------------------------------------------------------------- export { @@ -67,19 +73,26 @@ async function getAvatar (req: express.Request, res: express.Response) { const path = avatar.getPath() avatarPathUnsafeCache.set(filename, path) - return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER }) + return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) } async function getPreview (req: express.Request, res: express.Response) { const result = await VideosPreviewCache.Instance.getFilePath(req.params.filename) if (!result) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) } async function getVideoCaption (req: express.Request, res: express.Response) { const result = await VideosCaptionCache.Instance.getFilePath(req.params.filename) if (!result) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) +} + +async function getTorrent (req: express.Request, res: express.Response) { + const result = await VideosTorrentCache.Instance.getFilePath(req.params.filename) + if (!result) return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) } diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 2064857eb..7cc7f2c62 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -3,10 +3,7 @@ import * as express from 'express' import { join } from 'path' import { getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config' import { serveIndexHTML } from '@server/lib/client-html' -import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths' -import { MVideoFile, MVideoFullLight } from '@server/types/models' import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes' -import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type' import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo' import { root } from '../helpers/core-utils' import { CONFIG, isEmailEnabled } from '../initializers/config' @@ -16,14 +13,13 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, PEERTUBE_VERSION, ROUTE_CACHE_LIFETIME, - STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS, WEBSERVER } from '../initializers/constants' import { getThemeOrDefault } from '../lib/plugins/theme-utils' import { getEnabledResolutions } from '../lib/video-transcoding' -import { asyncMiddleware, videosDownloadValidator } from '../middlewares' +import { asyncMiddleware } from '../middlewares' import { cacheRoute } from '../middlewares/cache' import { UserModel } from '../models/account/user' import { VideoModel } from '../models/video/video' @@ -37,47 +33,23 @@ staticRouter.use(cors()) Cors is very important to let other servers access torrent and video files */ +// FIXME: deprecated in 3.2, use lazy-statics instead const torrentsPhysicalPath = CONFIG.STORAGE.TORRENTS_DIR staticRouter.use( STATIC_PATHS.TORRENTS, - cors(), express.static(torrentsPhysicalPath, { maxAge: 0 }) // Don't cache because we could regenerate the torrent file ) -staticRouter.use( - STATIC_DOWNLOAD_PATHS.TORRENTS + ':id-:resolution([0-9]+).torrent', - asyncMiddleware(videosDownloadValidator), - downloadTorrent -) -staticRouter.use( - STATIC_DOWNLOAD_PATHS.TORRENTS + ':id-:resolution([0-9]+)-hls.torrent', - asyncMiddleware(videosDownloadValidator), - downloadHLSVideoFileTorrent -) -// Videos path for webseeding +// Videos path for webseed staticRouter.use( STATIC_PATHS.WEBSEED, - cors(), express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }) // 404 because we don't have this video ) staticRouter.use( STATIC_PATHS.REDUNDANCY, - cors(), express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404 because we don't have this video ) -staticRouter.use( - STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', - asyncMiddleware(videosDownloadValidator), - downloadVideoFile -) - -staticRouter.use( - STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', - asyncMiddleware(videosDownloadValidator), - downloadHLSVideoFile -) - // HLS staticRouter.use( STATIC_PATHS.STREAMING_PLAYLISTS.HLS, @@ -327,60 +299,6 @@ async function generateNodeinfo (req: express.Request, res: express.Response) { return res.send(json).end() } -function downloadTorrent (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const videoFile = getVideoFile(req, video.VideoFiles) - if (!videoFile) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.download(getTorrentFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p.torrent`) -} - -function downloadHLSVideoFileTorrent (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const playlist = getHLSPlaylist(video) - if (!playlist) return res.status(HttpStatusCode.NOT_FOUND_404).end - - const videoFile = getVideoFile(req, playlist.VideoFiles) - if (!videoFile) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.download(getTorrentFilePath(playlist, videoFile), `${video.name}-${videoFile.resolution}p-hls.torrent`) -} - -function downloadVideoFile (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const videoFile = getVideoFile(req, video.VideoFiles) - if (!videoFile) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.download(getVideoFilePath(video, videoFile), `${video.name}-${videoFile.resolution}p${videoFile.extname}`) -} - -function downloadHLSVideoFile (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - const playlist = getHLSPlaylist(video) - if (!playlist) return res.status(HttpStatusCode.NOT_FOUND_404).end - - const videoFile = getVideoFile(req, playlist.VideoFiles) - if (!videoFile) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - const filename = `${video.name}-${videoFile.resolution}p-${playlist.getStringType()}${videoFile.extname}` - return res.download(getVideoFilePath(playlist, videoFile), filename) -} - -function getVideoFile (req: express.Request, files: MVideoFile[]) { - const resolution = parseInt(req.params.resolution, 10) - return files.find(f => f.resolution === resolution) -} - -function getHLSPlaylist (video: MVideoFullLight) { - const playlist = video.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) - if (!playlist) return undefined - - return Object.assign(playlist, { Video: video }) -} - function getCup (req: express.Request, res: express.Response, next: express.NextFunction) { res.status(HttpStatusCode.I_AM_A_TEAPOT_418) res.setHeader('Accept-Additions', 'Non-Dairy;1,Sugar;1') diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 1188d6cf9..02a9d4026 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -1,13 +1,13 @@ import * as Bluebird from 'bluebird' +import { URL } from 'url' import validator from 'validator' +import { ContextType } from '@shared/models/activitypub/context' import { ResultList } from '../../shared/models' import { Activity } from '../../shared/models/activitypub' import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' -import { signJsonLDObject } from './peertube-crypto' +import { MActor, MVideoWithHost } from '../types/models' import { pageToStartAndCount } from './core-utils' -import { URL } from 'url' -import { MActor, MVideoAccountLight } from '../types/models' -import { ContextType } from '@shared/models/activitypub/context' +import { signJsonLDObject } from './peertube-crypto' function getContextData (type: ContextType) { const context: any[] = [ @@ -201,8 +201,8 @@ function checkUrlsSameHost (url1: string, url2: string) { return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() } -function buildRemoteVideoBaseUrl (video: MVideoAccountLight, path: string) { - const host = video.VideoChannel.Account.Actor.Server.host +function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string) { + const host = video.VideoChannel.Actor.Server.host return REMOTE_SCHEME.HTTP + '://' + host + path } diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 9c5df2083..73418aa0a 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,20 +1,19 @@ -import { logger } from './logger' -import { generateVideoImportTmpPath } from './utils' -import * as WebTorrent from 'webtorrent' -import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra' -import { CONFIG } from '../initializers/config' -import { dirname, join } from 'path' import * as createTorrent from 'create-torrent' -import { promisify2 } from './core-utils' -import { MVideo } from '@server/types/models/video/video' -import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' -import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' -import { WEBSERVER } from '@server/initializers/constants' -import * as parseTorrent from 'parse-torrent' +import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra' import * as magnetUtil from 'magnet-uri' +import * as parseTorrent from 'parse-torrent' +import { dirname, join } from 'path' +import * as WebTorrent from 'webtorrent' import { isArray } from '@server/helpers/custom-validators/misc' -import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' -import { extractVideo } from '@server/helpers/video' +import { WEBSERVER } from '@server/initializers/constants' +import { generateTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' +import { MVideo, MVideoWithHost } from '@server/types/models/video/video' +import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' +import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' +import { CONFIG } from '../initializers/config' +import { promisify2 } from './core-utils' +import { logger } from './logger' +import { generateVideoImportTmpPath } from './utils' const createTorrentPromise = promisify2(createTorrent) @@ -78,10 +77,12 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName }) } -async function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - const video = extractVideo(videoOrPlaylist) - const { baseUrlHttp } = video.getBaseUrls() - +// FIXME: refactor/merge videoOrPlaylist and video arguments +async function createTorrentAndSetInfoHash ( + videoOrPlaylist: MVideo | MStreamingPlaylistVideo, + video: MVideoWithHost, + videoFile: MVideoFile +) { const options = { // Keep the extname, it's used by the client to stream the file inside a web browser name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`, @@ -90,33 +91,33 @@ async function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreaming [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], [ WEBSERVER.URL + '/tracker/announce' ] ], - urlList: [ videoOrPlaylist.getVideoFileUrl(videoFile, baseUrlHttp) ] + urlList: [ videoFile.getFileUrl(video) ] } const torrent = await createTorrentPromise(getVideoFilePath(videoOrPlaylist, videoFile), options) - const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, getTorrentFileName(videoOrPlaylist, videoFile)) - logger.info('Creating torrent %s.', filePath) + const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) + const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) + logger.info('Creating torrent %s.', torrentPath) - await writeFile(filePath, torrent) + await writeFile(torrentPath, torrent) const parsedTorrent = parseTorrent(torrent) videoFile.infoHash = parsedTorrent.infoHash + videoFile.torrentFilename = torrentFilename } +// FIXME: merge/refactor videoOrPlaylist and video arguments function generateMagnetUri ( videoOrPlaylist: MVideo | MStreamingPlaylistVideo, + video: MVideoWithHost, videoFile: MVideoFileRedundanciesOpt, baseUrlHttp: string, baseUrlWs: string ) { - const video = isStreamingPlaylist(videoOrPlaylist) - ? videoOrPlaylist.Video - : videoOrPlaylist - - const xs = videoOrPlaylist.getTorrentUrl(videoFile, baseUrlHttp) + const xs = videoFile.getTorrentUrl() const announce = videoOrPlaylist.getTrackerUrls(baseUrlHttp, baseUrlWs) - let urlList = [ videoOrPlaylist.getVideoFileUrl(videoFile, baseUrlHttp) ] + let urlList = [ videoFile.getFileUrl(video) ] const redundancies = videoFile.RedundancyVideos if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index a186afbdd..2578de5ed 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -17,7 +17,7 @@ function checkMissedConfig () { 'log.level', 'user.video_quota', 'user.video_quota_daily', 'csp.enabled', 'csp.report_only', 'csp.report_uri', - 'cache.previews.size', 'admin.email', 'contact_form.enabled', + 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'admin.email', 'contact_form.enabled', 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'redundancy.videos.strategies', 'redundancy.videos.check_interval', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 930fd784e..21ca78584 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -266,6 +266,9 @@ const CONFIG = { }, VIDEO_CAPTIONS: { get SIZE () { return config.get('cache.captions.size') } + }, + TORRENTS: { + get SIZE () { return config.get('cache.torrents.size') } } }, INSTANCE: { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index be5db8fe8..6b0984186 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -551,16 +551,13 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { // Express static paths (router) const STATIC_PATHS = { - PREVIEWS: '/static/previews/', THUMBNAILS: '/static/thumbnails/', TORRENTS: '/static/torrents/', WEBSEED: '/static/webseed/', REDUNDANCY: '/static/redundancy/', STREAMING_PLAYLISTS: { HLS: '/static/streaming-playlists/hls' - }, - AVATARS: '/static/avatars/', - VIDEO_CAPTIONS: '/static/video-captions/' + } } const STATIC_DOWNLOAD_PATHS = { TORRENTS: '/download/torrents/', @@ -570,12 +567,14 @@ const STATIC_DOWNLOAD_PATHS = { const LAZY_STATIC_PATHS = { AVATARS: '/lazy-static/avatars/', PREVIEWS: '/lazy-static/previews/', - VIDEO_CAPTIONS: '/lazy-static/video-captions/' + VIDEO_CAPTIONS: '/lazy-static/video-captions/', + TORRENTS: '/lazy-static/torrents/' } // Cache control const STATIC_MAX_AGE = { SERVER: '2h', + LAZY_SERVER: '2d', CLIENT: '30d' } @@ -609,6 +608,10 @@ const FILES_CACHE = { VIDEO_CAPTIONS: { DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'), MAX_AGE: 1000 * 3600 * 3 // 3 hours + }, + TORRENTS: { + DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'torrents'), + MAX_AGE: 1000 * 3600 * 3 // 3 hours } } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 66981f43f..a5f6537eb 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -1,7 +1,7 @@ import * as Bluebird from 'bluebird' import { maxBy, minBy } from 'lodash' import * as magnetUtil from 'magnet-uri' -import { join } from 'path' +import { basename, join } from 'path' import * as request from 'request' import * as sequelize from 'sequelize' import { VideoLiveModel } from '@server/models/video/video-live' @@ -30,11 +30,11 @@ import { doRequest } from '../../helpers/requests' import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' import { ACTIVITY_PUB, + LAZY_STATIC_PATHS, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, REMOTE_SCHEME, - STATIC_PATHS, THUMBNAILS_SIZE } from '../../initializers/constants' import { sequelizeTypescript } from '../../initializers/database' @@ -51,6 +51,8 @@ import { MChannelDefault, MChannelId, MStreamingPlaylist, + MStreamingPlaylistFilesVideo, + MStreamingPlaylistVideo, MVideo, MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, @@ -61,7 +63,8 @@ import { MVideoFullLight, MVideoId, MVideoImmutable, - MVideoThumbnail + MVideoThumbnail, + MVideoWithHost } from '../../types/models' import { MThumbnail } from '../../types/models/video/thumbnail' import { FilteredModelAttributes } from '../../types/sequelize' @@ -72,6 +75,7 @@ import { PeerTubeSocket } from '../peertube-socket' import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail' import { setVideoTags } from '../video' import { autoBlacklistVideoIfNeeded } from '../video-blacklist' +import { generateTorrentFileName } from '../video-paths' import { getOrCreateActorAndServerAndModel } from './actor' import { crawlCollectionPage } from './crawl' import { sendCreateVideo, sendUpdateVideo } from './send' @@ -405,7 +409,8 @@ async function updateVideoFromAP (options: { for (const playlistAttributes of streamingPlaylistAttributes) { const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t }) - .then(([ streamingPlaylist ]) => streamingPlaylist) + .then(([ streamingPlaylist ]) => streamingPlaylist as MStreamingPlaylistFilesVideo) + streamingPlaylistModel.Video = videoUpdated const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject) .map(a => new VideoFileModel(a)) @@ -637,13 +642,14 @@ async function createVideo (videoObject: VideoObject, channel: MChannelAccountLi videoCreated.VideoStreamingPlaylists = [] for (const playlistAttributes of streamingPlaylistsAttributes) { - const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) + const playlist = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t }) as MStreamingPlaylistFilesVideo + playlist.Video = videoCreated - const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject) + const playlistFiles = videoFileActivityUrlToDBAttributes(playlist, playlistAttributes.tagAPObject) const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t })) - playlistModel.VideoFiles = await Promise.all(videoFilePromises) + playlist.VideoFiles = await Promise.all(videoFilePromises) - videoCreated.VideoStreamingPlaylists.push(playlistModel) + videoCreated.VideoStreamingPlaylists.push(playlist) } // Process tags @@ -766,7 +772,7 @@ function videoActivityObjectToDBAttributes (videoChannel: MChannelId, videoObjec } function videoFileActivityUrlToDBAttributes ( - videoOrPlaylist: MVideo | MStreamingPlaylist, + videoOrPlaylist: MVideo | MStreamingPlaylistVideo, urls: (ActivityTagObject | ActivityUrlObject)[] ) { const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] @@ -786,6 +792,10 @@ function videoFileActivityUrlToDBAttributes ( throw new Error('Cannot parse magnet URI ' + magnet.href) } + const torrentUrl = Array.isArray(parsed.xs) + ? parsed.xs[0] + : parsed.xs + // Fetch associated metadata url, if any const metadata = urls.filter(isAPVideoFileMetadataObject) .find(u => { @@ -794,18 +804,30 @@ function videoFileActivityUrlToDBAttributes ( u.rel.includes(fileUrl.mediaType) }) - const mediaType = fileUrl.mediaType + const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) + const resolution = fileUrl.height + const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id + const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null + const attribute = { - extname: getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, mediaType), + extname, infoHash: parsed.infoHash, - resolution: fileUrl.height, + resolution, size: fileUrl.size, fps: fileUrl.fps || -1, metadataUrl: metadata?.href, + // Use the name of the remote file because we don't proxify video file requests + filename: basename(fileUrl.href), + fileUrl: fileUrl.href, + + torrentUrl, + // Use our own torrent name since we proxify torrent requests + torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), + // This is a video file owned by a video or by a streaming playlist - videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id, - videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null + videoId, + videoStreamingPlaylistId } attributes.push(attribute) @@ -862,8 +884,8 @@ function getPreviewFromIcons (videoObject: VideoObject) { return maxBy(validIcons, 'width') } -function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoAccountLight) { +function getPreviewUrl (previewIcon: ActivityIconObject, video: MVideoWithHost) { return previewIcon ? previewIcon.url - : buildRemoteVideoBaseUrl(video, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) + : buildRemoteVideoBaseUrl(video, join(LAZY_STATIC_PATHS.PREVIEWS, video.generatePreviewName())) } diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts index c06355446..af66689a0 100644 --- a/server/lib/files-cache/abstract-video-static-file-cache.ts +++ b/server/lib/files-cache/abstract-video-static-file-cache.ts @@ -2,7 +2,7 @@ import { remove } from 'fs-extra' import { logger } from '../../helpers/logger' import * as memoizee from 'memoizee' -type GetFilePathResult = { isOwned: boolean, path: string } | undefined +type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined export abstract class AbstractVideoStaticFileCache { diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/videos-torrent-cache.ts new file mode 100644 index 000000000..ca0e1770d --- /dev/null +++ b/server/lib/files-cache/videos-torrent-cache.ts @@ -0,0 +1,54 @@ +import { join } from 'path' +import { doRequestAndSaveToFile } from '@server/helpers/requests' +import { VideoFileModel } from '@server/models/video/video-file' +import { CONFIG } from '../../initializers/config' +import { FILES_CACHE } from '../../initializers/constants' +import { VideoModel } from '../../models/video/video' +import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' + +class VideosTorrentCache extends AbstractVideoStaticFileCache { + + private static instance: VideosTorrentCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(filename) + if (!file) return undefined + + if (file.getVideo().isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.TORRENTS_DIR, file.torrentFilename) } + + return this.loadRemoteFile(filename) + } + + // Key is the torrent filename + protected async loadRemoteFile (key: string) { + const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(key) + if (!file) return undefined + + if (file.getVideo().isOwned()) throw new Error('Cannot load remote file of owned video.') + + // Used to fetch the path + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.getVideo().id) + if (!video) return undefined + + const remoteUrl = file.getRemoteTorrentUrl(video) + const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) + + await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) + + const downloadName = `${video.name}-${file.resolution}p.torrent` + + return { isOwned: false, path: destPath, downloadName } + } +} + +export { + VideosTorrentCache +} diff --git a/server/lib/hls.ts b/server/lib/hls.ts index ef489097a..04187668c 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from import { sequelizeTypescript } from '../initializers/database' import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { getVideoFilename, getVideoFilePath } from './video-paths' +import { getVideoFilePath } from './video-paths' async function updateStreamingPlaylistsInfohashesIfNeeded () { const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() @@ -93,7 +93,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) { } await close(fd) - const videoFilename = getVideoFilename(hlsPlaylist, file) + const videoFilename = file.filename json[videoFilename] = rangeHashes } diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index cd95aa075..86c9b5c29 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -2,9 +2,9 @@ import * as Bull from 'bull' import { copy, stat } from 'fs-extra' import { extname } from 'path' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { getVideoFilePath } from '@server/lib/video-paths' +import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { UserModel } from '@server/models/account/user' -import { MVideoFile, MVideoWithFile } from '@server/types/models' +import { MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoFileImportPayload } from '@shared/models' import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils' import { logger } from '../../../helpers/logger' @@ -50,14 +50,16 @@ export { // --------------------------------------------------------------------------- -async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { +async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { const { videoFileResolution } = await getVideoFileResolution(inputFilePath) const { size } = await stat(inputFilePath) const fps = await getVideoFileFPS(inputFilePath) + const fileExt = extname(inputFilePath) let updatedVideoFile = new VideoFileModel({ resolution: videoFileResolution, - extname: extname(inputFilePath), + extname: fileExt, + filename: generateVideoFilename(video, false, videoFileResolution, fileExt), size, fps, videoId: video.id @@ -68,7 +70,7 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { if (currentVideoFile) { // Remove old file and old torrent await video.removeFile(currentVideoFile) - await video.removeTorrent(currentVideoFile) + await currentVideoFile.removeTorrent() // Remove the old video file from the array video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) @@ -83,7 +85,7 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) { const outputPath = getVideoFilePath(video, updatedVideoFile) await copy(inputFilePath, outputPath) - await createTorrentAndSetInfoHash(video, updatedVideoFile) + await createTorrentAndSetInfoHash(video, video, updatedVideoFile) await updatedVideoFile.save() diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 0d00c1b9d..8fa024105 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -6,7 +6,7 @@ import { isPostImportVideoAccepted } from '@server/lib/moderation' import { Hooks } from '@server/lib/plugins/hooks' import { isAbleToUploadVideo } from '@server/lib/user' import { addOptimizeOrMergeAudioJob } from '@server/lib/video' -import { getVideoFilePath } from '@server/lib/video-paths' +import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { ThumbnailModel } from '@server/models/video/thumbnail' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' import { @@ -116,10 +116,12 @@ async function processFile (downloader: () => Promise, videoImport: MVid const duration = await getDurationFromVideoFile(tempVideoPath) // Prepare video file object for creation in database + const fileExt = extname(tempVideoPath) const videoFileData = { - extname: extname(tempVideoPath), + extname: fileExt, resolution: videoFileResolution, size: stats.size, + filename: generateVideoFilename(videoImport.Video, false, videoFileResolution, fileExt), fps, videoId: videoImport.videoId } @@ -183,7 +185,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid } // Create torrent - await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) + await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoImportWithFiles.Video, videoFile) const videoFileSave = videoFile.toJSON() diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 6d50635bb..d57202ca5 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts @@ -85,7 +85,7 @@ async function saveLive (video: MVideo, live: MVideoLive) { await video.save() // Remove old HLS playlist video files - const videoWithFiles = await VideoModel.loadWithFiles(video.id) + const videoWithFiles = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id) const hlsPlaylist = videoWithFiles.getHLSPlaylist() await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index e248b645e..8573d4d12 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts @@ -128,7 +128,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay // Remove webtorrent files if not enabled for (const file of video.VideoFiles) { await video.removeFile(file) - await video.removeTorrent(file) + await file.removeTorrent() await file.destroy() } diff --git a/server/lib/live-manager.ts b/server/lib/live-manager.ts index 9f17b8820..b549c189f 100644 --- a/server/lib/live-manager.ts +++ b/server/lib/live-manager.ts @@ -16,7 +16,7 @@ import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' import { VideoLiveModel } from '@server/models/video/video-live' import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylist, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' +import { MStreamingPlaylist, MStreamingPlaylistVideo, MUserId, MVideoLive, MVideoLiveVideo } from '@server/types/models' import { VideoState, VideoStreamingPlaylistType } from '@shared/models' import { federateVideoIfNeeded } from './activitypub/videos' import { buildSha256Segment } from './hls' @@ -277,7 +277,7 @@ class LiveManager { return this.runMuxing({ sessionId, videoLive, - playlist: videoStreamingPlaylist, + playlist: Object.assign(videoStreamingPlaylist, { Video: video }), rtmpUrl, fps, allResolutions @@ -287,7 +287,7 @@ class LiveManager { private async runMuxing (options: { sessionId: string videoLive: MVideoLiveVideo - playlist: MStreamingPlaylist + playlist: MStreamingPlaylistVideo rtmpUrl: string fps: number allResolutions: number[] diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 93e76626c..60008e695 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -18,14 +18,14 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { logger } from '../../helpers/logger' import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { CONFIG } from '../../initializers/config' -import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' +import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos' import { downloadPlaylistSegments } from '../hls' import { removeVideoRedundancy } from '../redundancy' -import { getVideoFilename } from '../video-paths' +import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-paths' import { AbstractScheduler } from './abstract-scheduler' type CandidateToDuplicate = { @@ -222,17 +222,17 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) + const magnetUri = generateMagnetUri(video, video, file, baseUrlHttp, baseUrlWs) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, getVideoFilename(video, file)) + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) await move(tmpPath, destPath, { overwrite: true }) const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ expiresOn, url: getLocalVideoCacheFileActivityPubUrl(file), - fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), + fileUrl: generateWebTorrentRedundancyUrl(file), strategy, videoFileId: file.id, actorId: serverActor.id @@ -271,7 +271,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ expiresOn, url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), - fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), + fileUrl: generateHLSRedundancyUrl(video, playlistArg), strategy, videoStreamingPlaylistId: playlist.id, actorId: serverActor.id diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index 53fc8e81d..0385e89cc 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts @@ -1,19 +1,23 @@ -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' import { join } from 'path' -import { CONFIG } from '@server/initializers/config' -import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' import { extractVideo } from '@server/helpers/video' +import { CONFIG } from '@server/initializers/config' +import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' +import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' // ################## Video file name ################## -function getVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { +function generateVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, isHls: boolean, resolution: number, extname: string) { const video = extractVideo(videoOrPlaylist) - if (videoFile.isHLS()) { - return generateVideoStreamingPlaylistName(video.uuid, videoFile.resolution) + // FIXME: use a generated uuid instead, that will break compatibility with PeerTube < 3.2 + // const uuid = uuidv4() + const uuid = video.uuid + + if (isHls) { + return generateVideoStreamingPlaylistName(uuid, resolution) } - return generateWebTorrentVideoName(video.uuid, videoFile.resolution, videoFile.extname) + return generateWebTorrentVideoName(uuid, resolution, extname) } function generateVideoStreamingPlaylistName (uuid: string, resolution: number) { @@ -28,36 +32,64 @@ function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, vi if (videoFile.isHLS()) { const video = extractVideo(videoOrPlaylist) - return join(getHLSDirectory(video), getVideoFilename(videoOrPlaylist, videoFile)) + return join(getHLSDirectory(video), videoFile.filename) } - const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR - return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile)) + const baseDir = isRedundancy + ? CONFIG.STORAGE.REDUNDANCY_DIR + : CONFIG.STORAGE.VIDEOS_DIR + + return join(baseDir, videoFile.filename) +} + +// ################## Redundancy ################## + +function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { + // Base URL used by our HLS player + return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid +} + +function generateWebTorrentRedundancyUrl (file: MVideoFile) { + return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename } // ################## Streaming playlist ################## function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { - const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY + const baseDir = isRedundancy + ? HLS_REDUNDANCY_DIRECTORY + : HLS_STREAMING_PLAYLIST_DIRECTORY return join(baseDir, video.uuid) } // ################## Torrents ################## -function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { +function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { const video = extractVideo(videoOrPlaylist) const extension = '.torrent' + // FIXME: use a generated uuid instead, that will break compatibility with PeerTube < 3.2 + // const uuid = uuidv4() + const uuid = video.uuid + if (isStreamingPlaylist(videoOrPlaylist)) { - return `${video.uuid}-${videoFile.resolution}-${videoOrPlaylist.getStringType()}${extension}` + return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` } - return video.uuid + '-' + videoFile.resolution + extension + return uuid + '-' + resolution + extension +} + +function getTorrentFilePath (videoFile: MVideoFile) { + return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) } -function getTorrentFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - return join(CONFIG.STORAGE.TORRENTS_DIR, getTorrentFileName(videoOrPlaylist, videoFile)) +// ################## Meta data ################## + +function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { + const path = '/api/v1/videos/' + + return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id } // --------------------------------------------------------------------------- @@ -65,11 +97,16 @@ function getTorrentFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, export { generateVideoStreamingPlaylistName, generateWebTorrentVideoName, - getVideoFilename, + generateVideoFilename, getVideoFilePath, - getTorrentFileName, + generateTorrentFileName, getTorrentFilePath, - getHLSDirectory + getHLSDirectory, + + getLocalVideoFileMetadataUrl, + + generateWebTorrentRedundancyUrl, + generateHLSRedundancyUrl } diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts index a58c9dd20..b366e2e44 100644 --- a/server/lib/video-transcoding.ts +++ b/server/lib/video-transcoding.ts @@ -2,7 +2,7 @@ import { Job } from 'bull' import { copyFile, ensureDir, move, remove, stat } from 'fs-extra' import { basename, extname as extnameUtil, join } from 'path' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoWithAllFiles, MVideoWithFile } from '@server/types/models' +import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { VideoResolution } from '../../shared/models/videos' import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' import { transcode, TranscodeOptions, TranscodeOptionsType } from '../helpers/ffmpeg-utils' @@ -13,7 +13,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSER import { VideoFileModel } from '../models/video/video-file' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { updateMasterHLSPlaylist, updateSha256VODSegments } from './hls' -import { generateVideoStreamingPlaylistName, getVideoFilename, getVideoFilePath } from './video-paths' +import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from './video-paths' import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' /** @@ -24,7 +24,7 @@ import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' */ // Optimize the original video file and replace it. The resolution is not changed. -async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: MVideoFile, job?: Job) { +async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile: MVideoFile, job?: Job) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' @@ -55,8 +55,9 @@ async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: try { await remove(videoInputPath) - // Important to do this before getVideoFilename() to take in account the new file extension + // Important to do this before getVideoFilename() to take in account the new filename inputVideoFile.extname = newExtname + inputVideoFile.filename = generateVideoFilename(video, false, inputVideoFile.resolution, newExtname) const videoOutputPath = getVideoFilePath(video, inputVideoFile) @@ -72,7 +73,7 @@ async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFile: } // Transcode the original video file to a lower resolution. -async function transcodeNewWebTorrentResolution (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean, job: Job) { +async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolution: VideoResolution, isPortrait: boolean, job: Job) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const extname = '.mp4' @@ -82,11 +83,13 @@ async function transcodeNewWebTorrentResolution (video: MVideoWithFile, resoluti const newVideoFile = new VideoFileModel({ resolution, extname, + filename: generateVideoFilename(video, false, resolution, extname), size: 0, videoId: video.id }) + const videoOutputPath = getVideoFilePath(video, newVideoFile) - const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile)) + const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename) const transcodeOptions = resolution === VideoResolution.H_NOVIDEO ? { @@ -122,7 +125,7 @@ async function transcodeNewWebTorrentResolution (video: MVideoWithFile, resoluti } // Merge an image with an audio file to create a video -async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution, job: Job) { +async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoResolution, job: Job) { const transcodeDirectory = CONFIG.STORAGE.TMP_DIR const newExtname = '.mp4' @@ -175,7 +178,7 @@ async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: Video // Concat TS segments from a live video to a fragmented mp4 HLS playlist async function generateHlsPlaylistResolutionFromTS (options: { - video: MVideoWithFile + video: MVideoFullLight concatenatedTsFilePath: string resolution: VideoResolution isPortraitMode: boolean @@ -193,7 +196,7 @@ async function generateHlsPlaylistResolutionFromTS (options: { // Generate an HLS playlist from an input file, and update the master playlist function generateHlsPlaylistResolution (options: { - video: MVideoWithFile + video: MVideoFullLight videoInputPath: string resolution: VideoResolution copyCodecs: boolean @@ -235,7 +238,7 @@ export { // --------------------------------------------------------------------------- async function onWebTorrentVideoFileTranscoding ( - video: MVideoWithFile, + video: MVideoFullLight, videoFile: MVideoFile, transcodingPath: string, outputPath: string @@ -250,7 +253,7 @@ async function onWebTorrentVideoFileTranscoding ( videoFile.fps = fps videoFile.metadata = metadata - await createTorrentAndSetInfoHash(video, videoFile) + await createTorrentAndSetInfoHash(video, video, videoFile) await VideoFileModel.customUpsert(videoFile, 'video', undefined) video.VideoFiles = await video.$get('VideoFiles') @@ -260,7 +263,7 @@ async function onWebTorrentVideoFileTranscoding ( async function generateHlsPlaylistCommon (options: { type: 'hls' | 'hls-from-ts' - video: MVideoWithFile + video: MVideoFullLight inputPath: string resolution: VideoResolution copyCodecs?: boolean @@ -318,10 +321,12 @@ async function generateHlsPlaylistCommon (options: { videoStreamingPlaylist.Video = video // Build the new playlist file + const extname = extnameUtil(videoFilename) const newVideoFile = new VideoFileModel({ resolution, - extname: extnameUtil(videoFilename), + extname, size: 0, + filename: generateVideoFilename(video, true, resolution, extname), fps: -1, videoStreamingPlaylistId: videoStreamingPlaylist.id }) @@ -344,7 +349,7 @@ async function generateHlsPlaylistCommon (options: { newVideoFile.fps = await getVideoFileFPS(videoFilePath) newVideoFile.metadata = await getMetadataFromFile(videoFilePath) - await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile) + await createTorrentAndSetInfoHash(videoStreamingPlaylist, video, newVideoFile) await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined) videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles') diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index 4185ec5f2..9533c8d19 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -17,7 +17,7 @@ import { } from 'sequelize-typescript' import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { MThumbnail, MThumbnailVideo, MVideoAccountLight } from '@server/types/models' +import { MThumbnail, MThumbnailVideo, MVideoWithHost } from '@server/types/models' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { logger } from '../../helpers/logger' import { CONFIG } from '../../initializers/config' @@ -164,7 +164,7 @@ export class ThumbnailModel extends Model { return join(directory, filename) } - getFileUrl (video: MVideoAccountLight) { + getFileUrl (video: MVideoWithHost) { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename if (video.isOwned()) return WEBSERVER.URL + staticPath diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index a1553ea15..71b067335 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -15,8 +15,9 @@ import { Table, UpdatedAt } from 'sequelize-typescript' +import { v4 as uuidv4 } from 'uuid' import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' -import { MVideoAccountLight, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models' +import { MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo, MVideoWithHost } from '@server/types/models' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' import { logger } from '../../helpers/logger' @@ -24,7 +25,6 @@ import { CONFIG } from '../../initializers/config' import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, VIDEO_LANGUAGES, WEBSERVER } from '../../initializers/constants' import { buildWhereIdOrUUID, throwIfNotValid } from '../utils' import { VideoModel } from './video' -import { v4 as uuidv4 } from 'uuid' export enum ScopeNames { WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE' @@ -204,7 +204,7 @@ export class VideoCaptionModel extends Model { return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename) } - getFileUrl (video: MVideoAccountLight) { + getFileUrl (video: MVideoWithHost) { if (!this.Video) this.Video = video as VideoModel if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 48b337c68..57807cbfd 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -1,3 +1,7 @@ +import { remove } from 'fs-extra' +import * as memoizee from 'memoizee' +import { join } from 'path' +import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' import { AllowNull, BelongsTo, @@ -5,15 +9,22 @@ import { CreatedAt, DataType, Default, + DefaultScope, ForeignKey, HasMany, Is, Model, - Table, - UpdatedAt, Scopes, - DefaultScope + Table, + UpdatedAt } from 'sequelize-typescript' +import { Where } from 'sequelize/types/lib/utils' +import validator from 'validator' +import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub' +import { logger } from '@server/helpers/logger' +import { extractVideo } from '@server/helpers/video' +import { getTorrentFilePath } from '@server/lib/video-paths' +import { MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' import { isVideoFileExtnameValid, isVideoFileInfoHashValid, @@ -21,20 +32,25 @@ import { isVideoFileSizeValid, isVideoFPSResolutionValid } from '../../helpers/custom-validators/videos' +import { + LAZY_STATIC_PATHS, + MEMOIZE_LENGTH, + MEMOIZE_TTL, + MIMETYPES, + STATIC_DOWNLOAD_PATHS, + STATIC_PATHS, + WEBSERVER +} from '../../initializers/constants' +import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { parseAggregateResult, throwIfNotValid } from '../utils' import { VideoModel } from './video' -import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' -import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' -import { MIMETYPES, MEMOIZE_LENGTH, MEMOIZE_TTL } from '../../initializers/constants' -import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file' -import { MStreamingPlaylistVideo, MVideo } from '@server/types/models' -import * as memoizee from 'memoizee' -import validator from 'validator' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO', - WITH_METADATA = 'WITH_METADATA' + WITH_METADATA = 'WITH_METADATA', + WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST' } @DefaultScope(() => ({ @@ -51,6 +67,28 @@ export enum ScopeNames { } ] }, + [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: Where } = {}) => { + return { + include: [ + { + model: VideoModel.unscoped(), + required: false, + where: options.whereVideo + }, + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: options.whereVideo + } + ] + } + ] + } + }, [ScopeNames.WITH_METADATA]: { attributes: { include: [ 'metadata' ] @@ -81,6 +119,16 @@ export enum ScopeNames { fields: [ 'infoHash' ] }, + { + fields: [ 'torrentFilename' ], + unique: true + }, + + { + fields: [ 'filename' ], + unique: true + }, + { fields: [ 'videoId', 'resolution', 'fps' ], unique: true, @@ -142,6 +190,24 @@ export class VideoFileModel extends Model { @Column metadataUrl: string + @AllowNull(true) + @Column + fileUrl: string + + // Could be null for live files + @AllowNull(true) + @Column + filename: string + + @AllowNull(true) + @Column + torrentUrl: string + + // Could be null for live files + @AllowNull(true) + @Column + torrentFilename: string + @ForeignKey(() => VideoModel) @Column videoId: number @@ -199,6 +265,16 @@ export class VideoFileModel extends Model { return !!videoFile } + static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { + const query = { + where: { + torrentFilename: filename + } + } + + return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) + } + static loadWithMetadata (id: number) { return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) } @@ -215,28 +291,11 @@ export class VideoFileModel extends Model { const options = { where: { id - }, - include: [ - { - model: VideoModel.unscoped(), - required: false, - where: whereVideo - }, - { - model: VideoStreamingPlaylistModel.unscoped(), - required: false, - include: [ - { - model: VideoModel.unscoped(), - required: true, - where: whereVideo - } - ] - } - ] + } } - return VideoFileModel.findOne(options) + return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] }) + .findOne(options) .then(file => { // We used `required: false` so check we have at least a video or a streaming playlist if (!file.Video && !file.VideoStreamingPlaylist) return null @@ -348,6 +407,10 @@ export class VideoFileModel extends Model { return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist } + getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo { + return extractVideo(this.getVideoOrStreamingPlaylist()) + } + isAudio () { return !!MIMETYPES.AUDIO.EXT_MIMETYPE[this.extname] } @@ -360,6 +423,62 @@ export class VideoFileModel extends Model { return !!this.videoStreamingPlaylistId } + getFileUrl (video: MVideoWithHost) { + if (!this.Video) this.Video = video as VideoModel + + if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) + if (this.fileUrl) return this.fileUrl + + // Fallback if we don't have a file URL + return buildRemoteVideoBaseUrl(video, this.getFileStaticPath(video)) + } + + getFileStaticPath (video: MVideo) { + if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) + + return join(STATIC_PATHS.WEBSEED, this.filename) + } + + getFileDownloadUrl (video: MVideoWithHost) { + const basePath = this.isHLS() + ? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + : STATIC_DOWNLOAD_PATHS.VIDEOS + const path = join(basePath, this.filename) + + if (video.isOwned()) return WEBSERVER.URL + path + + // FIXME: don't guess remote URL + return buildRemoteVideoBaseUrl(video, path) + } + + getRemoteTorrentUrl (video: MVideoWithHost) { + if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) + + if (this.torrentUrl) return this.torrentUrl + + // Fallback if we don't have a torrent URL + return buildRemoteVideoBaseUrl(video, this.getTorrentStaticPath()) + } + + // We proxify torrent requests so use a local URL + getTorrentUrl () { + return WEBSERVER.URL + this.getTorrentStaticPath() + } + + getTorrentStaticPath () { + return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename) + } + + getTorrentDownloadUrl () { + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename) + } + + removeTorrent () { + const torrentPath = getTorrentFilePath(this) + return remove(torrentPath) + .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) + } + hasSameUniqueKeysThan (other: MVideoFile) { return this.fps === other.fps && this.resolution === other.resolution && diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 77b8bcfe3..adf460734 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -1,16 +1,17 @@ -import { Video, VideoDetails } from '../../../shared/models/videos' -import { VideoModel } from './video' +import { generateMagnetUri } from '@server/helpers/webtorrent' +import { getLocalVideoFileMetadataUrl } from '@server/lib/video-paths' +import { VideoFile } from '@shared/models/videos/video-file.model' import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../shared/models/activitypub/objects' +import { Video, VideoDetails } from '../../../shared/models/videos' +import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' +import { isArray } from '../../helpers/custom-validators/misc' import { MIMETYPES, WEBSERVER } from '../../initializers/constants' -import { VideoCaptionModel } from './video-caption' import { getLocalVideoCommentsActivityPubUrl, getLocalVideoDislikesActivityPubUrl, getLocalVideoLikesActivityPubUrl, getLocalVideoSharesActivityPubUrl } from '../../lib/activitypub/url' -import { isArray } from '../../helpers/custom-validators/misc' -import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model' import { MStreamingPlaylistRedundanciesOpt, MStreamingPlaylistVideo, @@ -18,12 +19,12 @@ import { MVideoAP, MVideoFile, MVideoFormattable, - MVideoFormattableDetails + MVideoFormattableDetails, + MVideoWithHost } from '../../types/models' import { MVideoFileRedundanciesOpt } from '../../types/models/video/video-file' -import { VideoFile } from '@shared/models/videos/video-file.model' -import { generateMagnetUri } from '@server/helpers/webtorrent' -import { extractVideo } from '@server/helpers/video' +import { VideoModel } from './video' +import { VideoCaptionModel } from './video-caption' export type VideoFormattingJSONOptions = { completeDescription?: boolean @@ -153,12 +154,15 @@ function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): Vid } // Format and sort video files - detailsJson.files = videoFilesModelToFormattedJSON(video, baseUrlHttp, baseUrlWs, video.VideoFiles) + detailsJson.files = videoFilesModelToFormattedJSON(video, video, baseUrlHttp, baseUrlWs, video.VideoFiles) return Object.assign(formattedJson, detailsJson) } -function streamingPlaylistsModelToFormattedJSON (video: MVideo, playlists: MStreamingPlaylistRedundanciesOpt[]): VideoStreamingPlaylist[] { +function streamingPlaylistsModelToFormattedJSON ( + video: MVideoFormattableDetails, + playlists: MStreamingPlaylistRedundanciesOpt[] +): VideoStreamingPlaylist[] { if (isArray(playlists) === false) return [] const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() @@ -171,7 +175,7 @@ function streamingPlaylistsModelToFormattedJSON (video: MVideo, playlists: MStre ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) : [] - const files = videoFilesModelToFormattedJSON(playlistWithVideo, baseUrlHttp, baseUrlWs, playlist.VideoFiles) + const files = videoFilesModelToFormattedJSON(playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles) return { id: playlist.id, @@ -190,14 +194,14 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) { return -1 } +// FIXME: refactor/merge model and video arguments function videoFilesModelToFormattedJSON ( model: MVideo | MStreamingPlaylistVideo, + video: MVideoFormattableDetails, baseUrlHttp: string, baseUrlWs: string, videoFiles: MVideoFileRedundanciesOpt[] ): VideoFile[] { - const video = extractVideo(model) - return [ ...videoFiles ] .filter(f => !f.isLive()) .sort(sortByResolutionDesc) @@ -207,21 +211,29 @@ function videoFilesModelToFormattedJSON ( id: videoFile.resolution, label: videoFile.resolution + 'p' }, - magnetUri: generateMagnetUri(model, videoFile, baseUrlHttp, baseUrlWs), + + // FIXME: deprecated in 3.2 + magnetUri: generateMagnetUri(model, video, videoFile, baseUrlHttp, baseUrlWs), + size: videoFile.size, fps: videoFile.fps, - torrentUrl: model.getTorrentUrl(videoFile, baseUrlHttp), - torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp), - fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp), - fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp), - metadataUrl: video.getVideoFileMetadataUrl(videoFile, baseUrlHttp) + + torrentUrl: videoFile.getTorrentUrl(), + torrentDownloadUrl: videoFile.getTorrentDownloadUrl(), + + fileUrl: videoFile.getFileUrl(video), + fileDownloadUrl: videoFile.getFileDownloadUrl(video), + + metadataUrl: videoFile.metadataUrl ?? getLocalVideoFileMetadataUrl(video, videoFile) } as VideoFile }) } +// FIXME: refactor/merge model and video arguments function addVideoFilesInAPAcc ( acc: ActivityUrlObject[] | ActivityTagObject[], model: MVideoAP | MStreamingPlaylistVideo, + video: MVideoWithHost, baseUrlHttp: string, baseUrlWs: string, files: MVideoFile[] @@ -234,7 +246,7 @@ function addVideoFilesInAPAcc ( acc.push({ type: 'Link', mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] as any, - href: model.getVideoFileUrl(file, baseUrlHttp), + href: file.getFileUrl(video), height: file.resolution, size: file.size, fps: file.fps @@ -244,7 +256,7 @@ function addVideoFilesInAPAcc ( type: 'Link', rel: [ 'metadata', MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] ], mediaType: 'application/json' as 'application/json', - href: extractVideo(model).getVideoFileMetadataUrl(file, baseUrlHttp), + href: getLocalVideoFileMetadataUrl(video, file), height: file.resolution, fps: file.fps }) @@ -252,14 +264,14 @@ function addVideoFilesInAPAcc ( acc.push({ type: 'Link', mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', - href: model.getTorrentUrl(file, baseUrlHttp), + href: file.getTorrentUrl(), height: file.resolution }) acc.push({ type: 'Link', mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: generateMagnetUri(model, file, baseUrlHttp, baseUrlWs), + href: generateMagnetUri(model, video, file, baseUrlHttp, baseUrlWs), height: file.resolution }) } @@ -307,7 +319,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { } ] - addVideoFilesInAPAcc(url, video, baseUrlHttp, baseUrlWs, video.VideoFiles || []) + addVideoFilesInAPAcc(url, video, video, baseUrlHttp, baseUrlWs, video.VideoFiles || []) for (const playlist of (video.VideoStreamingPlaylists || [])) { const tag = playlist.p2pMediaLoaderInfohashes @@ -320,7 +332,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { }) const playlistWithVideo = Object.assign(playlist, { Video: video }) - addVideoFilesInAPAcc(tag, playlistWithVideo, baseUrlHttp, baseUrlWs, playlist.VideoFiles || []) + addVideoFilesInAPAcc(tag, playlistWithVideo, video, baseUrlHttp, baseUrlWs, playlist.VideoFiles || []) url.push({ type: 'Link', diff --git a/server/models/video/video-query-builder.ts b/server/models/video/video-query-builder.ts index 822d0c89b..af1878e7a 100644 --- a/server/models/video/video-query-builder.ts +++ b/server/models/video/video-query-builder.ts @@ -516,6 +516,10 @@ function wrapForAPIResults (baseQuery: string, replacements: any, options: Build '"VideoFiles"."resolution"': '"VideoFiles.resolution"', '"VideoFiles"."size"': '"VideoFiles.size"', '"VideoFiles"."extname"': '"VideoFiles.extname"', + '"VideoFiles"."filename"': '"VideoFiles.filename"', + '"VideoFiles"."fileUrl"': '"VideoFiles.fileUrl"', + '"VideoFiles"."torrentFilename"': '"VideoFiles.torrentFilename"', + '"VideoFiles"."torrentUrl"': '"VideoFiles.torrentUrl"', '"VideoFiles"."infoHash"': '"VideoFiles.infoHash"', '"VideoFiles"."fps"': '"VideoFiles.fps"', '"VideoFiles"."videoId"': '"VideoFiles.videoId"', @@ -529,6 +533,10 @@ function wrapForAPIResults (baseQuery: string, replacements: any, options: Build '"VideoStreamingPlaylists->VideoFiles"."resolution"': '"VideoStreamingPlaylists.VideoFiles.resolution"', '"VideoStreamingPlaylists->VideoFiles"."size"': '"VideoStreamingPlaylists.VideoFiles.size"', '"VideoStreamingPlaylists->VideoFiles"."extname"': '"VideoStreamingPlaylists.VideoFiles.extname"', + '"VideoStreamingPlaylists->VideoFiles"."filename"': '"VideoStreamingPlaylists.VideoFiles.filename"', + '"VideoStreamingPlaylists->VideoFiles"."fileUrl"': '"VideoStreamingPlaylists.VideoFiles.fileUrl"', + '"VideoStreamingPlaylists->VideoFiles"."torrentFilename"': '"VideoStreamingPlaylists.VideoFiles.torrentFilename"', + '"VideoStreamingPlaylists->VideoFiles"."torrentUrl"': '"VideoStreamingPlaylists.VideoFiles.torrentUrl"', '"VideoStreamingPlaylists->VideoFiles"."infoHash"': '"VideoStreamingPlaylists.VideoFiles.infoHash"', '"VideoStreamingPlaylists->VideoFiles"."fps"': '"VideoStreamingPlaylists.VideoFiles.fps"', '"VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId"': '"VideoStreamingPlaylists.VideoFiles.videoStreamingPlaylistId"', diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index 148768c21..c9375b433 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -1,28 +1,18 @@ +import * as memoizee from 'memoizee' +import { join } from 'path' +import { Op, QueryTypes } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { throwIfNotValid } from '../utils' -import { VideoModel } from './video' -import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { VideoFileModel } from '@server/models/video/video-file' +import { MStreamingPlaylist } from '@server/types/models' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { - CONSTRAINTS_FIELDS, - MEMOIZE_LENGTH, - MEMOIZE_TTL, - P2P_MEDIA_LOADER_PEER_VERSION, - STATIC_DOWNLOAD_PATHS, - STATIC_PATHS -} from '../../initializers/constants' -import { join } from 'path' import { sha1 } from '../../helpers/core-utils' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { isArrayOf } from '../../helpers/custom-validators/misc' -import { Op, QueryTypes } from 'sequelize' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' -import { VideoFileModel } from '@server/models/video/video-file' -import { getTorrentFileName, getTorrentFilePath, getVideoFilename } from '@server/lib/video-paths' -import * as memoizee from 'memoizee' -import { remove } from 'fs-extra' -import { logger } from '@server/helpers/logger' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' +import { throwIfNotValid } from '../utils' +import { VideoModel } from './video' @Table({ tableName: 'videoStreamingPlaylist', @@ -196,26 +186,6 @@ export class VideoStreamingPlaylistModel extends Model { return 'unknown' } - getVideoRedundancyUrl (baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid - } - - getTorrentDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + getTorrentFileName(this, videoFile) - } - - getVideoFileDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + getVideoFilename(this, videoFile) - } - - getVideoFileUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, this.Video.uuid, getVideoFilename(this, videoFile)) - } - - getTorrentUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + join(STATIC_PATHS.TORRENTS, getTorrentFileName(this, videoFile)) - } - getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] } @@ -224,10 +194,4 @@ export class VideoStreamingPlaylistModel extends Model { return this.type === other.type && this.videoId === other.videoId } - - removeTorrent (this: MStreamingPlaylistVideo, videoFile: MVideoFile) { - const torrentPath = getTorrentFilePath(this, videoFile) - return remove(torrentPath) - .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) - } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 3321deed3..2e6b6aeec 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -24,10 +24,11 @@ import { Table, UpdatedAt } from 'sequelize-typescript' +import { v4 as uuidv4 } from 'uuid' import { buildNSFWFilter } from '@server/helpers/express-utils' import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' import { LiveManager } from '@server/lib/live-manager' -import { getHLSDirectory, getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { getHLSDirectory, getVideoFilePath } from '@server/lib/video-paths' import { getServerActor } from '@server/models/application/application' import { ModelCache } from '@server/models/model-cache' import { VideoFile } from '@shared/models/videos/video-file.model' @@ -60,7 +61,6 @@ import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, REMOTE_SCHEME, - STATIC_DOWNLOAD_PATHS, STATIC_PATHS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, @@ -78,6 +78,7 @@ import { MStreamingPlaylistFilesVideo, MUserAccountId, MUserId, + MVideo, MVideoAccountLight, MVideoAccountLightBlacklistAllFiles, MVideoAP, @@ -130,7 +131,6 @@ import { VideoShareModel } from './video-share' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' import { VideoTagModel } from './video-tag' import { VideoViewModel } from './video-view' -import { v4 as uuidv4 } from 'uuid' export enum ScopeNames { AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS', @@ -790,7 +790,7 @@ export class VideoModel extends Model { // Remove physical files and torrents instance.VideoFiles.forEach(file => { tasks.push(instance.removeFile(file)) - tasks.push(instance.removeTorrent(file)) + tasks.push(file.removeTorrent()) }) // Remove playlists file @@ -853,18 +853,14 @@ export class VideoModel extends Model { return undefined } - static listLocal (): Promise { + static listLocal (): Promise { const query = { where: { remote: false } } - return VideoModel.scope([ - ScopeNames.WITH_WEBTORRENT_FILES, - ScopeNames.WITH_STREAMING_PLAYLISTS, - ScopeNames.WITH_THUMBNAILS - ]).findAll(query) + return VideoModel.findAll(query) } static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { @@ -1623,6 +1619,10 @@ export class VideoModel extends Model { 'resolution', 'size', 'extname', + 'filename', + 'fileUrl', + 'torrentFilename', + 'torrentUrl', 'infoHash', 'fps', 'videoId', @@ -1891,14 +1891,14 @@ export class VideoModel extends Model { let files: VideoFile[] = [] if (Array.isArray(this.VideoFiles)) { - const result = videoFilesModelToFormattedJSON(this, baseUrlHttp, baseUrlWs, this.VideoFiles) + const result = videoFilesModelToFormattedJSON(this, this, baseUrlHttp, baseUrlWs, this.VideoFiles) files = files.concat(result) } for (const p of (this.VideoStreamingPlaylists || [])) { p.Video = this - const result = videoFilesModelToFormattedJSON(p, baseUrlHttp, baseUrlWs, p.VideoFiles) + const result = videoFilesModelToFormattedJSON(p, this, baseUrlHttp, baseUrlWs, p.VideoFiles) files = files.concat(result) } @@ -1956,12 +1956,6 @@ export class VideoModel extends Model { .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) } - removeTorrent (videoFile: MVideoFile) { - const torrentPath = getTorrentFilePath(this, videoFile) - return remove(torrentPath) - .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) - } - async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { const directoryPath = getHLSDirectory(this, isRedundancy) @@ -1977,7 +1971,7 @@ export class VideoModel extends Model { // Remove physical files and torrents await Promise.all( - streamingPlaylistWithFiles.VideoFiles.map(file => streamingPlaylistWithFiles.removeTorrent(file)) + streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) ) } } @@ -2054,34 +2048,6 @@ export class VideoModel extends Model { return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] } - getTorrentUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.TORRENTS + getTorrentFileName(this, videoFile) - } - - getTorrentDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + getTorrentFileName(this, videoFile) - } - - getVideoFileUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.WEBSEED + getVideoFilename(this, videoFile) - } - - getVideoFileMetadataUrl (videoFile: MVideoFile, baseUrlHttp: string) { - const path = '/api/v1/videos/' - - return this.isOwned() - ? baseUrlHttp + path + this.uuid + '/metadata/' + videoFile.id - : videoFile.metadataUrl - } - - getVideoRedundancyUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_PATHS.REDUNDANCY + getVideoFilename(this, videoFile) - } - - getVideoFileDownloadUrl (videoFile: MVideoFile, baseUrlHttp: string) { - return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + getVideoFilename(this, videoFile) - } - getBandwidthBits (videoFile: MVideoFile) { return Math.ceil((videoFile.size * 8) / this.duration) } diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index db551dd9e..03ac3f321 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -52,7 +52,7 @@ async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string, hlsOn expect(file).to.not.be.undefined expect(file.magnetUri).to.have.lengthOf.above(2) - expect(file.torrentUrl).to.equal(`${baseUrl}/static/torrents/${videoDetails.uuid}-${file.resolution.id}-hls.torrent`) + expect(file.torrentUrl).to.equal(`http://${server.host}/lazy-static/torrents/${videoDetails.uuid}-${file.resolution.id}-hls.torrent`) expect(file.fileUrl).to.equal( `${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${videoDetails.uuid}-${file.resolution.id}-fragmented.mp4` ) diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index dac049fe4..7eaf2c19e 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts @@ -2,7 +2,7 @@ import 'mocha' import * as chai from 'chai' -import { VideoDetails } from '../../../shared/models/videos' +import { VideoFile } from '@shared/models/videos/video-file.model' import { cleanupTests, doubleFollow, @@ -16,7 +16,7 @@ import { uploadVideo } from '../../../shared/extra-utils' import { waitJobs } from '../../../shared/extra-utils/server/jobs' -import { VideoFile } from '@shared/models/videos/video-file.model' +import { VideoDetails } from '../../../shared/models/videos' const expect = chai.expect @@ -62,7 +62,6 @@ describe('Test create import video jobs', function () { await waitJobs(servers) - let magnetUri: string for (const server of servers) { const { data: videos } = (await getVideosList(server.url)).body expect(videos).to.have.lengthOf(2) @@ -74,9 +73,6 @@ describe('Test create import video jobs', function () { const [ originalVideo, transcodedVideo ] = videoDetail.files assertVideoProperties(originalVideo, 720, 'webm', 218910) assertVideoProperties(transcodedVideo, 480, 'webm', 69217) - - if (!magnetUri) magnetUri = transcodedVideo.magnetUri - else expect(transcodedVideo.magnetUri).to.equal(magnetUri) } }) @@ -86,7 +82,6 @@ describe('Test create import video jobs', function () { await waitJobs(servers) - let magnetUri: string for (const server of servers) { const { data: videos } = (await getVideosList(server.url)).body expect(videos).to.have.lengthOf(2) @@ -100,9 +95,6 @@ describe('Test create import video jobs', function () { assertVideoProperties(transcodedVideo420, 480, 'mp4') assertVideoProperties(transcodedVideo320, 360, 'mp4') assertVideoProperties(transcodedVideo240, 240, 'mp4') - - if (!magnetUri) magnetUri = originalVideo.magnetUri - else expect(originalVideo.magnetUri).to.equal(magnetUri) } }) @@ -112,7 +104,6 @@ describe('Test create import video jobs', function () { await waitJobs(servers) - let magnetUri: string for (const server of servers) { const { data: videos } = (await getVideosList(server.url)).body expect(videos).to.have.lengthOf(2) @@ -124,9 +115,6 @@ describe('Test create import video jobs', function () { const [ video720, video480 ] = videoDetail.files assertVideoProperties(video720, 720, 'webm', 942961) assertVideoProperties(video480, 480, 'webm', 69217) - - if (!magnetUri) magnetUri = video720.magnetUri - else expect(video720.magnetUri).to.equal(magnetUri) } }) diff --git a/server/types/models/video/video-channels.ts b/server/types/models/video/video-channels.ts index 2e05d8753..77790daa4 100644 --- a/server/types/models/video/video-channels.ts +++ b/server/types/models/video/video-channels.ts @@ -17,6 +17,7 @@ import { MActorDefault, MActorDefaultLight, MActorFormattable, + MActorHost, MActorLight, MActorSummary, MActorSummaryFormattable, MActorUrl @@ -71,6 +72,10 @@ export type MChannelAccountLight = Use<'Actor', MActorDefaultLight> & Use<'Account', MAccountLight> +export type MChannelHost = + MChannelId & + Use<'Actor', MActorHost> + // ############################################################################ // Account associations diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts index ae23cc30f..92dcbaf59 100644 --- a/server/types/models/video/video.ts +++ b/server/types/models/video/video.ts @@ -1,27 +1,28 @@ -import { VideoModel } from '../../../models/video/video' import { PickWith, PickWithOpt } from '@shared/core-utils' +import { VideoModel } from '../../../models/video/video' +import { MUserVideoHistoryTime } from '../user/user-video-history' +import { MScheduleVideoUpdate } from './schedule-video-update' +import { MTag } from './tag' +import { MThumbnail } from './thumbnail' +import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' +import { MVideoCaptionLanguage, MVideoCaptionLanguageUrl } from './video-caption' import { MChannelAccountDefault, MChannelAccountLight, MChannelAccountSummaryFormattable, MChannelActor, MChannelFormattable, + MChannelHost, MChannelUserId } from './video-channels' -import { MTag } from './tag' -import { MVideoCaptionLanguage, MVideoCaptionLanguageUrl } from './video-caption' +import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file' +import { MVideoLive } from './video-live' import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesAll, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' -import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file' -import { MThumbnail } from './thumbnail' -import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' -import { MScheduleVideoUpdate } from './schedule-video-update' -import { MUserVideoHistoryTime } from '../user/user-video-history' -import { MVideoLive } from './video-live' type Use = PickWith @@ -143,6 +144,10 @@ export type MVideoWithChannelActor = MVideo & Use<'VideoChannel', MChannelActor> +export type MVideoWithHost = + MVideo & + Use<'VideoChannel', MChannelHost> + export type MVideoFullLight = MVideo & Use<'Thumbnails', MThumbnail[]> & diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index f94fa233c..929eb42ca 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -11,7 +11,7 @@ import validator from 'validator' import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' import { VideoDetails, VideoPrivacy } from '../../models/videos' import { buildAbsoluteFixturePath, buildServerDirectory, dateIsValid, immutableAssign, testImage, webtorrentAdd } from '../miscs/miscs' -import { makeGetRequest, makePutBodyRequest, makeUploadRequest } from '../requests/requests' +import { makeGetRequest, makePutBodyRequest, makeRawRequest, makeUploadRequest } from '../requests/requests' import { waitJobs } from '../server/jobs' import { ServerInfo } from '../server/servers' import { getMyUserInformation } from '../users/users' @@ -544,6 +544,9 @@ async function completeVideoCheck ( if (!attributes.likes) attributes.likes = 0 if (!attributes.dislikes) attributes.dislikes = 0 + const host = new URL(url).host + const originHost = attributes.account.host + expect(video.name).to.equal(attributes.name) expect(video.category.id).to.equal(attributes.category) expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Misc') @@ -603,8 +606,21 @@ async function completeVideoCheck ( if (attributes.files.length > 1) extension = '.mp4' expect(file.magnetUri).to.have.lengthOf.above(2) - expect(file.torrentUrl).to.equal(`http://${attributes.account.host}/static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`) - expect(file.fileUrl).to.equal(`http://${attributes.account.host}/static/webseed/${videoDetails.uuid}-${file.resolution.id}${extension}`) + + expect(file.torrentDownloadUrl).to.equal(`http://${host}/download/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`) + expect(file.torrentUrl).to.equal(`http://${host}/lazy-static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`) + + expect(file.fileUrl).to.equal(`http://${originHost}/static/webseed/${videoDetails.uuid}-${file.resolution.id}${extension}`) + expect(file.fileDownloadUrl).to.equal(`http://${originHost}/download/videos/${videoDetails.uuid}-${file.resolution.id}${extension}`) + + await Promise.all([ + makeRawRequest(file.torrentUrl, 200), + makeRawRequest(file.torrentDownloadUrl, 200), + makeRawRequest(file.metadataUrl, 200), + // Backward compatibility + makeRawRequest(`http://${originHost}/static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`, 200) + ]) + expect(file.resolution.id).to.equal(attributeFile.resolution) expect(file.resolution.label).to.equal(attributeFile.resolution + 'p') diff --git a/shared/models/videos/video-file.model.ts b/shared/models/videos/video-file.model.ts index e9839229d..f3d93f0ed 100644 --- a/shared/models/videos/video-file.model.ts +++ b/shared/models/videos/video-file.model.ts @@ -3,14 +3,20 @@ import { VideoFileMetadata } from './video-file-metadata' import { VideoResolution } from './video-resolution.enum' export interface VideoFile { - magnetUri: string resolution: VideoConstant size: number // Bytes + torrentUrl: string torrentDownloadUrl: string + fileUrl: string fileDownloadUrl: string + fps: number + metadata?: VideoFileMetadata metadataUrl?: string + + // FIXME: deprecated in 3.2 + magnetUri: string } -- 2.41.0