From 83903cb65d531a6b6b91715387493ba8312b264d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 22 Jul 2021 14:28:03 +0200 Subject: [PATCH] Generate random uuid for video files --- server/controllers/api/videos/upload.ts | 4 +- server/lib/hls.ts | 6 +- .../job-queue/handlers/video-file-import.ts | 4 +- server/lib/job-queue/handlers/video-import.ts | 4 +- server/lib/transcoding/video-transcoding.ts | 12 +- server/lib/video-paths.ts | 35 ++--- server/tests/api/check-params/abuses.ts | 20 +-- server/tests/api/check-params/services.ts | 16 +- .../tests/api/check-params/video-blacklist.ts | 42 +++-- server/tests/api/moderation/abuses.ts | 54 +++---- server/tests/api/redundancy/redundancy.ts | 145 ++++++++++-------- .../tests/api/users/users-multiple-servers.ts | 20 +-- server/tests/api/videos/audio-only.ts | 12 +- server/tests/api/videos/multiple-servers.ts | 21 +-- server/tests/api/videos/resumable-upload.ts | 8 +- server/tests/api/videos/single-server.ts | 3 +- server/tests/api/videos/video-captions.ts | 5 +- .../api/videos/video-change-ownership.ts | 16 +- server/tests/api/videos/video-hls.ts | 17 +- server/tests/api/videos/video-transcoder.ts | 60 +++++--- server/tests/plugins/plugin-helpers.ts | 4 +- shared/core-utils/miscs/index.ts | 1 + shared/core-utils/miscs/regexp.ts | 1 + shared/extra-utils/server/server.ts | 18 +-- shared/extra-utils/server/servers-command.ts | 9 ++ .../extra-utils/videos/streaming-playlists.ts | 4 +- shared/extra-utils/videos/videos.ts | 89 +++++++---- 27 files changed, 364 insertions(+), 266 deletions(-) create mode 100644 shared/core-utils/miscs/regexp.ts diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 1603ef127..7792ae3fc 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -6,7 +6,7 @@ import { uuidToShort } from '@server/helpers/uuid' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' -import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { openapiOperationDoc } from '@server/middlewares/doc' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' import { uploadx } from '@uploadx/core' @@ -240,7 +240,7 @@ async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUplo videoFile.resolution = (await getVideoFileResolution(videoPhysicalFile.path)).videoFileResolution } - videoFile.filename = generateVideoFilename(video, false, videoFile.resolution, videoFile.extname) + videoFile.filename = generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) return videoFile } diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 05be403f3..212bd095b 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -36,8 +36,10 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) { const streamingPlaylist = video.getHLSPlaylist() for (const file of streamingPlaylist.VideoFiles) { + const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution) + // If we did not generated a playlist for this resolution, skip - const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + const filePlaylistPath = join(directory, playlistFilename) if (await pathExists(filePlaylistPath) === false) continue const videoFilePath = getVideoFilePath(streamingPlaylist, file) @@ -58,7 +60,7 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) { line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` masterPlaylists.push(line) - masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) + masterPlaylists.push(playlistFilename) } await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 187cb652e..1783f206a 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -2,7 +2,7 @@ import * as Bull from 'bull' import { copy, stat } from 'fs-extra' import { getLowercaseExtension } from '@server/helpers/core-utils' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { UserModel } from '@server/models/user/user' import { MVideoFullLight } from '@server/types/models' import { VideoFileImportPayload } from '@shared/models' @@ -72,7 +72,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { const newVideoFile = new VideoFileModel({ resolution: videoFileResolution, extname: fileExt, - filename: generateVideoFilename(video, false, videoFileResolution, fileExt), + filename: generateWebTorrentVideoFilename(videoFileResolution, fileExt), size, fps, videoId: video.id diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 55498003d..6e425d09c 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -8,7 +8,7 @@ import { Hooks } from '@server/lib/plugins/hooks' import { ServerConfigManager } from '@server/lib/server-config-manager' import { isAbleToUploadVideo } from '@server/lib/user' import { addOptimizeOrMergeAudioJob } from '@server/lib/video' -import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { generateWebTorrentVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { ThumbnailModel } from '@server/models/video/thumbnail' import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' import { @@ -124,7 +124,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid extname: fileExt, resolution: videoFileResolution, size: stats.size, - filename: generateVideoFilename(videoImport.Video, false, videoFileResolution, fileExt), + filename: generateWebTorrentVideoFilename(videoFileResolution, fileExt), fps, videoId: videoImport.videoId } diff --git a/server/lib/transcoding/video-transcoding.ts b/server/lib/transcoding/video-transcoding.ts index 1ad63baf3..d70f7f474 100644 --- a/server/lib/transcoding/video-transcoding.ts +++ b/server/lib/transcoding/video-transcoding.ts @@ -14,7 +14,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSER import { VideoFileModel } from '../../models/video/video-file' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls' -import { generateVideoFilename, generateVideoStreamingPlaylistName, getVideoFilePath } from '../video-paths' +import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths' import { VideoTranscodingProfilesManager } from './video-transcoding-profiles' /** @@ -60,7 +60,7 @@ async function optimizeOriginalVideofile (video: MVideoFullLight, inputVideoFile // Important to do this before getVideoFilename() to take in account the new filename inputVideoFile.extname = newExtname - inputVideoFile.filename = generateVideoFilename(video, false, resolution, newExtname) + inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) const videoOutputPath = getVideoFilePath(video, inputVideoFile) @@ -86,7 +86,7 @@ async function transcodeNewWebTorrentResolution (video: MVideoFullLight, resolut const newVideoFile = new VideoFileModel({ resolution, extname, - filename: generateVideoFilename(video, false, resolution, extname), + filename: generateWebTorrentVideoFilename(resolution, extname), size: 0, videoId: video.id }) @@ -169,7 +169,7 @@ async function mergeAudioVideofile (video: MVideoFullLight, resolution: VideoRes // Important to do this before getVideoFilename() to take in account the new file extension inputVideoFile.extname = newExtname - inputVideoFile.filename = generateVideoFilename(video, false, inputVideoFile.resolution, newExtname) + inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) const videoOutputPath = getVideoFilePath(video, inputVideoFile) // ffmpeg generated a new video file, so update the video duration @@ -271,7 +271,7 @@ async function generateHlsPlaylistCommon (options: { const videoTranscodedBasePath = join(transcodeDirectory, type) await ensureDir(videoTranscodedBasePath) - const videoFilename = generateVideoStreamingPlaylistName(video.uuid, resolution) + const videoFilename = generateHLSVideoFilename(resolution) const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution) const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename) @@ -319,7 +319,7 @@ async function generateHlsPlaylistCommon (options: { resolution, extname, size: 0, - filename: generateVideoFilename(video, true, resolution, extname), + filename: videoFilename, fps: -1, videoStreamingPlaylistId: videoStreamingPlaylist.id }) diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index 1708c479a..b7068190c 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts @@ -3,29 +3,20 @@ import { extractVideo } from '@server/helpers/video' import { CONFIG } from '@server/initializers/config' import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' +import { buildUUID } from '@server/helpers/uuid' // ################## Video file name ################## -function generateVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, isHls: boolean, resolution: number, extname: string) { - const video = extractVideo(videoOrPlaylist) +function generateWebTorrentVideoFilename (resolution: number, extname: string) { + const uuid = buildUUID() - // FIXME: use a generated uuid instead, that will break compatibility with PeerTube < 3.1 - // const uuid = uuidv4() - const uuid = video.uuid - - if (isHls) { - return generateVideoStreamingPlaylistName(uuid, resolution) - } - - return generateWebTorrentVideoName(uuid, resolution, extname) + return uuid + '-' + resolution + extname } -function generateVideoStreamingPlaylistName (uuid: string, resolution: number) { - return `${uuid}-${resolution}-fragmented.mp4` -} +function generateHLSVideoFilename (resolution: number) { + const uuid = buildUUID() -function generateWebTorrentVideoName (uuid: string, resolution: number, extname: string) { - return uuid + '-' + resolution + extname + return `${uuid}-${resolution}-fragmented.mp4` } function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) { @@ -66,12 +57,8 @@ function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { // ################## Torrents ################## function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { - const video = extractVideo(videoOrPlaylist) const extension = '.torrent' - - // FIXME: use a generated uuid instead, that will break compatibility with PeerTube < 3.1 - // const uuid = uuidv4() - const uuid = video.uuid + const uuid = buildUUID() if (isStreamingPlaylist(videoOrPlaylist)) { return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` @@ -95,9 +82,9 @@ function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) // --------------------------------------------------------------------------- export { - generateVideoStreamingPlaylistName, - generateWebTorrentVideoName, - generateVideoFilename, + generateHLSVideoFilename, + generateWebTorrentVideoFilename, + getVideoFilePath, generateTorrentFileName, diff --git a/server/tests/api/check-params/abuses.ts b/server/tests/api/check-params/abuses.ts index 72f2cbd8f..fb9a5fd8b 100644 --- a/server/tests/api/check-params/abuses.ts +++ b/server/tests/api/check-params/abuses.ts @@ -41,7 +41,7 @@ describe('Test abuses API validators', function () { userToken = await server.users.generateUserAndToken('user_1') userToken2 = await server.users.generateUserAndToken('user_2') - server.store.video = await server.videos.upload() + server.store.videoCreated = await server.videos.upload() command = server.abuses }) @@ -223,25 +223,25 @@ describe('Test abuses API validators', function () { }) it('Should fail with a non authenticated user', async function () { - const fields = { video: { id: server.store.video.id }, reason: 'my super reason' } + const fields = { video: { id: server.store.videoCreated.id }, reason: 'my super reason' } await makePostBodyRequest({ url: server.url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) }) it('Should fail with a reason too short', async function () { - const fields = { video: { id: server.store.video.id }, reason: 'h' } + const fields = { video: { id: server.store.videoCreated.id }, reason: 'h' } await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) }) it('Should fail with a too big reason', async function () { - const fields = { video: { id: server.store.video.id }, reason: 'super'.repeat(605) } + const fields = { video: { id: server.store.videoCreated.id }, reason: 'super'.repeat(605) } await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) }) it('Should succeed with the correct parameters (basic)', async function () { - const fields: AbuseCreate = { video: { id: server.store.video.shortUUID }, reason: 'my super reason' } + const fields: AbuseCreate = { video: { id: server.store.videoCreated.shortUUID }, reason: 'my super reason' } const res = await makePostBodyRequest({ url: server.url, @@ -254,19 +254,19 @@ describe('Test abuses API validators', function () { }) it('Should fail with a wrong predefined reason', async function () { - const fields = { video: { id: server.store.video.id }, reason: 'my super reason', predefinedReasons: [ 'wrongPredefinedReason' ] } + const fields = { video: server.store.videoCreated, reason: 'my super reason', predefinedReasons: [ 'wrongPredefinedReason' ] } await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) }) it('Should fail with negative timestamps', async function () { - const fields = { video: { id: server.store.video.id, startAt: -1 }, reason: 'my super reason' } + const fields = { video: { id: server.store.videoCreated.id, startAt: -1 }, reason: 'my super reason' } await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) }) it('Should fail mith misordered startAt/endAt', async function () { - const fields = { video: { id: server.store.video.id, startAt: 5, endAt: 1 }, reason: 'my super reason' } + const fields = { video: { id: server.store.videoCreated.id, startAt: 5, endAt: 1 }, reason: 'my super reason' } await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) }) @@ -274,7 +274,7 @@ describe('Test abuses API validators', function () { it('Should succeed with the corret parameters (advanced)', async function () { const fields: AbuseCreate = { video: { - id: server.store.video.id, + id: server.store.videoCreated.id, startAt: 1, endAt: 5 }, @@ -413,7 +413,7 @@ describe('Test abuses API validators', function () { await doubleFollow(anotherServer, server) - const server2VideoId = await anotherServer.videos.getId({ uuid: server.store.video.uuid }) + const server2VideoId = await anotherServer.videos.getId({ uuid: server.store.videoCreated.uuid }) await anotherServer.abuses.report({ reason: 'remote server', videoId: server2VideoId }) await waitJobs([ server, anotherServer ]) diff --git a/server/tests/api/check-params/services.ts b/server/tests/api/check-params/services.ts index 4c4a5cade..8d795fabc 100644 --- a/server/tests/api/check-params/services.ts +++ b/server/tests/api/check-params/services.ts @@ -24,7 +24,7 @@ describe('Test services API validators', function () { await setAccessTokensToServers([ server ]) await setDefaultVideoChannel([ server ]) - server.store.video = await server.videos.upload({ attributes: { name: 'my super name' } }) + server.store.videoCreated = await server.videos.upload({ attributes: { name: 'my super name' } }) { const created = await server.playlists.create({ @@ -47,7 +47,7 @@ describe('Test services API validators', function () { }) it('Should fail with an invalid host', async function () { - const embedUrl = 'http://hello.com/videos/watch/' + server.store.video.uuid + const embedUrl = 'http://hello.com/videos/watch/' + server.store.videoCreated.uuid await checkParamEmbed(server, embedUrl) }) @@ -62,37 +62,37 @@ describe('Test services API validators', function () { }) it('Should fail with an invalid path', async function () { - const embedUrl = `http://localhost:${server.port}/videos/watchs/${server.store.video.uuid}` + const embedUrl = `http://localhost:${server.port}/videos/watchs/${server.store.videoCreated.uuid}` await checkParamEmbed(server, embedUrl) }) it('Should fail with an invalid max height', async function () { - const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.video.uuid}` + const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.videoCreated.uuid}` await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { maxheight: 'hello' }) }) it('Should fail with an invalid max width', async function () { - const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.video.uuid}` + const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.videoCreated.uuid}` await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { maxwidth: 'hello' }) }) it('Should fail with an invalid format', async function () { - const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.video.uuid}` + const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.videoCreated.uuid}` await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { format: 'blabla' }) }) it('Should fail with a non supported format', async function () { - const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.video.uuid}` + const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.videoCreated.uuid}` await checkParamEmbed(server, embedUrl, HttpStatusCode.NOT_IMPLEMENTED_501, { format: 'xml' }) }) it('Should succeed with the correct params with a video', async function () { - const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.video.uuid}` + const embedUrl = `http://localhost:${server.port}/videos/watch/${server.store.videoCreated.uuid}` const query = { format: 'json', maxheight: 400, diff --git a/server/tests/api/check-params/video-blacklist.ts b/server/tests/api/check-params/video-blacklist.ts index d28c6a952..1f926d227 100644 --- a/server/tests/api/check-params/video-blacklist.ts +++ b/server/tests/api/check-params/video-blacklist.ts @@ -51,7 +51,7 @@ describe('Test video blacklist API validators', function () { } { - servers[0].store.video = await servers[0].videos.upload({ token: userAccessToken1 }) + servers[0].store.videoCreated = await servers[0].videos.upload({ token: userAccessToken1 }) } { @@ -73,7 +73,7 @@ describe('Test video blacklist API validators', function () { const basePath = '/api/v1/videos/' it('Should fail with nothing', async function () { - const path = basePath + servers[0].store.video + '/blacklist' + const path = basePath + servers[0].store.videoCreated + '/blacklist' const fields = {} await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) }) @@ -85,13 +85,13 @@ describe('Test video blacklist API validators', function () { }) it('Should fail with a non authenticated user', async function () { - const path = basePath + servers[0].store.video + '/blacklist' + const path = basePath + servers[0].store.videoCreated + '/blacklist' const fields = {} await makePostBodyRequest({ url: servers[0].url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) }) it('Should fail with a non admin user', async function () { - const path = basePath + servers[0].store.video + '/blacklist' + const path = basePath + servers[0].store.videoCreated + '/blacklist' const fields = {} await makePostBodyRequest({ url: servers[0].url, @@ -103,7 +103,7 @@ describe('Test video blacklist API validators', function () { }) it('Should fail with an invalid reason', async function () { - const path = basePath + servers[0].store.video.uuid + '/blacklist' + const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' const fields = { reason: 'a'.repeat(305) } await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) @@ -123,7 +123,7 @@ describe('Test video blacklist API validators', function () { }) it('Should succeed with the correct params', async function () { - const path = basePath + servers[0].store.video.uuid + '/blacklist' + const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' const fields = {} await makePostBodyRequest({ @@ -158,13 +158,13 @@ describe('Test video blacklist API validators', function () { }) it('Should fail with a non authenticated user', async function () { - const path = basePath + servers[0].store.video + '/blacklist' + const path = basePath + servers[0].store.videoCreated + '/blacklist' const fields = {} await makePutBodyRequest({ url: servers[0].url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) }) it('Should fail with a non admin user', async function () { - const path = basePath + servers[0].store.video + '/blacklist' + const path = basePath + servers[0].store.videoCreated + '/blacklist' const fields = {} await makePutBodyRequest({ url: servers[0].url, @@ -176,14 +176,14 @@ describe('Test video blacklist API validators', function () { }) it('Should fail with an invalid reason', async function () { - const path = basePath + servers[0].store.video.uuid + '/blacklist' + const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' const fields = { reason: 'a'.repeat(305) } await makePutBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) }) it('Should succeed with the correct params', async function () { - const path = basePath + servers[0].store.video.shortUUID + '/blacklist' + const path = basePath + servers[0].store.videoCreated.shortUUID + '/blacklist' const fields = { reason: 'hello' } await makePutBodyRequest({ @@ -199,24 +199,24 @@ describe('Test video blacklist API validators', function () { describe('When getting blacklisted video', function () { it('Should fail with a non authenticated user', async function () { - await servers[0].videos.get({ id: servers[0].store.video.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await servers[0].videos.get({ id: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) }) it('Should fail with another user', async function () { await servers[0].videos.getWithToken({ token: userAccessToken2, - id: servers[0].store.video.uuid, + id: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) }) it('Should succeed with the owner authenticated user', async function () { - const video = await servers[0].videos.getWithToken({ token: userAccessToken1, id: servers[0].store.video.uuid }) + const video = await servers[0].videos.getWithToken({ token: userAccessToken1, id: servers[0].store.videoCreated.uuid }) expect(video.blacklisted).to.be.true }) it('Should succeed with an admin', async function () { - const video = servers[0].store.video + const video = servers[0].store.videoCreated for (const id of [ video.id, video.uuid, video.shortUUID ]) { const video = await servers[0].videos.getWithToken({ id, expectedStatus: HttpStatusCode.OK_200 }) @@ -228,11 +228,19 @@ describe('Test video blacklist API validators', function () { describe('When removing a video in blacklist', function () { it('Should fail with a non authenticated user', async function () { - await command.remove({ token: 'fake token', videoId: servers[0].store.video.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await command.remove({ + token: 'fake token', + videoId: servers[0].store.videoCreated.uuid, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) }) it('Should fail with a non admin user', async function () { - await command.remove({ token: userAccessToken2, videoId: servers[0].store.video.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await command.remove({ + token: userAccessToken2, + videoId: servers[0].store.videoCreated.uuid, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) }) it('Should fail with an incorrect id', async function () { @@ -245,7 +253,7 @@ describe('Test video blacklist API validators', function () { }) it('Should succeed with the correct params', async function () { - await command.remove({ videoId: servers[0].store.video.uuid, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + await command.remove({ videoId: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) }) }) diff --git a/server/tests/api/moderation/abuses.ts b/server/tests/api/moderation/abuses.ts index 8d6360eb3..c258414ce 100644 --- a/server/tests/api/moderation/abuses.ts +++ b/server/tests/api/moderation/abuses.ts @@ -64,8 +64,8 @@ describe('Test abuses', function () { const { data } = await servers[0].videos.list() expect(data.length).to.equal(2) - servers[0].store.video = data.find(video => video.name === 'my super name for server 1') - servers[1].store.video = data.find(video => video.name === 'my super name for server 2') + servers[0].store.videoCreated = data.find(video => video.name === 'my super name for server 1') + servers[1].store.videoCreated = data.find(video => video.name === 'my super name for server 2') }) it('Should not have abuses', async function () { @@ -80,7 +80,7 @@ describe('Test abuses', function () { this.timeout(15000) const reason = 'my super bad reason' - await commands[0].report({ videoId: servers[0].store.video.id, reason }) + await commands[0].report({ videoId: servers[0].store.videoCreated.id, reason }) // We wait requests propagation, even if the server 1 is not supposed to make a request to server 2 await waitJobs(servers) @@ -100,7 +100,7 @@ describe('Test abuses', function () { expect(abuse.reporterAccount.name).to.equal('root') expect(abuse.reporterAccount.host).to.equal(servers[0].host) - expect(abuse.video.id).to.equal(servers[0].store.video.id) + expect(abuse.video.id).to.equal(servers[0].store.videoCreated.id) expect(abuse.video.channel).to.exist expect(abuse.comment).to.be.null @@ -127,7 +127,7 @@ describe('Test abuses', function () { this.timeout(10000) const reason = 'my super bad reason 2' - const videoId = await servers[0].videos.getId({ uuid: servers[1].store.video.uuid }) + const videoId = await servers[0].videos.getId({ uuid: servers[1].store.videoCreated.uuid }) await commands[0].report({ videoId, reason }) // We wait requests propagation @@ -146,7 +146,7 @@ describe('Test abuses', function () { expect(abuse1.reporterAccount.name).to.equal('root') expect(abuse1.reporterAccount.host).to.equal(servers[0].host) - expect(abuse1.video.id).to.equal(servers[0].store.video.id) + expect(abuse1.video.id).to.equal(servers[0].store.videoCreated.id) expect(abuse1.video.countReports).to.equal(1) expect(abuse1.video.nthReport).to.equal(1) @@ -165,7 +165,7 @@ describe('Test abuses', function () { expect(abuse2.reporterAccount.name).to.equal('root') expect(abuse2.reporterAccount.host).to.equal(servers[0].host) - expect(abuse2.video.id).to.equal(servers[1].store.video.id) + expect(abuse2.video.id).to.equal(servers[1].store.videoCreated.id) expect(abuse2.comment).to.be.null @@ -200,7 +200,7 @@ describe('Test abuses', function () { this.timeout(10000) { - const videoId = await servers[1].videos.getId({ uuid: servers[0].store.video.uuid }) + const videoId = await servers[1].videos.getId({ uuid: servers[0].store.videoCreated.uuid }) await commands[1].report({ videoId, reason: 'will mute this' }) await waitJobs(servers) @@ -288,7 +288,7 @@ describe('Test abuses', function () { await commands[0].report({ videoId: video3Id, reason: reason3 }) const reason4 = 'my super bad reason 4' - await commands[0].report({ token: userAccessToken, videoId: servers[0].store.video.id, reason: reason4 }) + await commands[0].report({ token: userAccessToken, videoId: servers[0].store.videoCreated.id, reason: reason4 }) { const body = await commands[0].getAdminList() @@ -301,7 +301,7 @@ describe('Test abuses', function () { expect(abuseVideo3.countReportsForReportee).to.equal(1, "wrong reports count for reporter on video 3 abuse") expect(abuseVideo3.countReportsForReporter).to.equal(3, "wrong reports count for reportee on video 3 abuse") - const abuseServer1 = abuses.find(a => a.video.id === servers[0].store.video.id) + const abuseServer1 = abuses.find(a => a.video.id === servers[0].store.videoCreated.id) expect(abuseServer1.countReportsForReportee).to.equal(3, "wrong reports count for reporter on video 1 abuse") } }) @@ -312,7 +312,7 @@ describe('Test abuses', function () { const reason5 = 'my super bad reason 5' const predefinedReasons5: AbusePredefinedReasonsString[] = [ 'violentOrRepulsive', 'captions' ] const createRes = await commands[0].report({ - videoId: servers[0].store.video.id, + videoId: servers[0].store.videoCreated.id, reason: reason5, predefinedReasons: predefinedReasons5, startAt: 1, @@ -402,11 +402,11 @@ describe('Test abuses', function () { before(async function () { this.timeout(50000) - servers[0].store.video = await servers[0].videos.quickUpload({ name: 'server 1' }) - servers[1].store.video = await servers[1].videos.quickUpload({ name: 'server 2' }) + servers[0].store.videoCreated = await servers[0].videos.quickUpload({ name: 'server 1' }) + servers[1].store.videoCreated = await servers[1].videos.quickUpload({ name: 'server 2' }) - await servers[0].comments.createThread({ videoId: servers[0].store.video.id, text: 'comment server 1' }) - await servers[1].comments.createThread({ videoId: servers[1].store.video.id, text: 'comment server 2' }) + await servers[0].comments.createThread({ videoId: servers[0].store.videoCreated.id, text: 'comment server 1' }) + await servers[1].comments.createThread({ videoId: servers[1].store.videoCreated.id, text: 'comment server 2' }) await waitJobs(servers) }) @@ -414,7 +414,7 @@ describe('Test abuses', function () { it('Should report abuse on a comment', async function () { this.timeout(15000) - const comment = await getComment(servers[0], servers[0].store.video.id) + const comment = await getComment(servers[0], servers[0].store.videoCreated.id) const reason = 'it is a bad comment' await commands[0].report({ commentId: comment.id, reason }) @@ -424,7 +424,7 @@ describe('Test abuses', function () { it('Should have 1 comment abuse on server 1 and 0 on server 2', async function () { { - const comment = await getComment(servers[0], servers[0].store.video.id) + const comment = await getComment(servers[0], servers[0].store.videoCreated.id) const body = await commands[0].getAdminList({ filter: 'comment' }) expect(body.total).to.equal(1) @@ -442,8 +442,8 @@ describe('Test abuses', function () { expect(abuse.comment.id).to.equal(comment.id) expect(abuse.comment.text).to.equal(comment.text) expect(abuse.comment.video.name).to.equal('server 1') - expect(abuse.comment.video.id).to.equal(servers[0].store.video.id) - expect(abuse.comment.video.uuid).to.equal(servers[0].store.video.uuid) + expect(abuse.comment.video.id).to.equal(servers[0].store.videoCreated.id) + expect(abuse.comment.video.uuid).to.equal(servers[0].store.videoCreated.uuid) expect(abuse.countReportsForReporter).to.equal(5) expect(abuse.countReportsForReportee).to.equal(5) @@ -459,7 +459,7 @@ describe('Test abuses', function () { it('Should report abuse on a remote comment', async function () { this.timeout(10000) - const comment = await getComment(servers[0], servers[1].store.video.uuid) + const comment = await getComment(servers[0], servers[1].store.videoCreated.uuid) const reason = 'it is a really bad comment' await commands[0].report({ commentId: comment.id, reason }) @@ -468,7 +468,7 @@ describe('Test abuses', function () { }) it('Should have 2 comment abuses on server 1 and 1 on server 2', async function () { - const commentServer2 = await getComment(servers[0], servers[1].store.video.id) + const commentServer2 = await getComment(servers[0], servers[1].store.videoCreated.id) { const body = await commands[0].getAdminList({ filter: 'comment' }) @@ -493,7 +493,7 @@ describe('Test abuses', function () { expect(abuse2.comment.id).to.equal(commentServer2.id) expect(abuse2.comment.text).to.equal(commentServer2.text) expect(abuse2.comment.video.name).to.equal('server 2') - expect(abuse2.comment.video.uuid).to.equal(servers[1].store.video.uuid) + expect(abuse2.comment.video.uuid).to.equal(servers[1].store.videoCreated.uuid) expect(abuse2.state.id).to.equal(AbuseState.PENDING) expect(abuse2.state.label).to.equal('Pending') @@ -527,9 +527,9 @@ describe('Test abuses', function () { it('Should keep the comment abuse when deleting the comment', async function () { this.timeout(10000) - const commentServer2 = await getComment(servers[0], servers[1].store.video.id) + const commentServer2 = await getComment(servers[0], servers[1].store.videoCreated.id) - await servers[0].comments.delete({ videoId: servers[1].store.video.uuid, commentId: commentServer2.id }) + await servers[0].comments.delete({ videoId: servers[1].store.videoCreated.uuid, commentId: commentServer2.id }) await waitJobs(servers) @@ -761,9 +761,9 @@ describe('Test abuses', function () { before(async function () { userAccessToken = await servers[0].users.generateUserAndToken('user_42') - await commands[0].report({ token: userAccessToken, videoId: servers[0].store.video.id, reason: 'user reason 1' }) + await commands[0].report({ token: userAccessToken, videoId: servers[0].store.videoCreated.id, reason: 'user reason 1' }) - const videoId = await servers[0].videos.getId({ uuid: servers[1].store.video.uuid }) + const videoId = await servers[0].videos.getId({ uuid: servers[1].store.videoCreated.uuid }) await commands[0].report({ token: userAccessToken, videoId, reason: 'user reason 2' }) }) @@ -832,7 +832,7 @@ describe('Test abuses', function () { before(async function () { userToken = await servers[0].users.generateUserAndToken('user_43') - const body = await commands[0].report({ token: userToken, videoId: servers[0].store.video.id, reason: 'user 43 reason 1' }) + const body = await commands[0].report({ token: userToken, videoId: servers[0].store.videoCreated.id, reason: 'user 43 reason 1' }) abuseId = body.abuse.id }) diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index a6559d304..e1a12f5f8 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -4,7 +4,7 @@ import 'mocha' import * as chai from 'chai' import { readdir } from 'fs-extra' import * as magnetUtil from 'magnet-uri' -import { join } from 'path' +import { basename, join } from 'path' import { checkSegmentHash, checkVideoFilesWereRemoved, @@ -12,30 +12,41 @@ import { createMultipleServers, doubleFollow, killallServers, - makeGetRequest, + makeRawRequest, PeerTubeServer, root, + saveVideoInServers, setAccessTokensToServers, wait, waitJobs } from '@shared/extra-utils' -import { HttpStatusCode, VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models' +import { + HttpStatusCode, + VideoDetails, + VideoFile, + VideoPrivacy, + VideoRedundancyStrategy, + VideoRedundancyStrategyWithManual +} from '@shared/models' const expect = chai.expect let servers: PeerTubeServer[] = [] -let video1Server2UUID: string -let video1Server2Id: number +let video1Server2: VideoDetails -function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: PeerTubeServer) { +async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) { const parsed = magnetUtil.decode(file.magnetUri) for (const ws of baseWebseeds) { - const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) + const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`) expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined } expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) + + for (const url of parsed.urlList) { + await makeRawRequest(url, HttpStatusCode.OK_200) + } } async function createSingleServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) { @@ -76,11 +87,10 @@ async function createSingleServers (strategy: VideoRedundancyStrategy | null, ad await setAccessTokensToServers(servers) { - const { uuid, id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) - video1Server2UUID = uuid - video1Server2Id = id + const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) + video1Server2 = await servers[1].videos.get({ id }) - await servers[1].videos.view({ id: video1Server2UUID }) + await servers[1].videos.view({ id }) } await waitJobs(servers) @@ -95,11 +105,33 @@ async function createSingleServers (strategy: VideoRedundancyStrategy | null, ad await waitJobs(servers) } +async function ensureSameFilenames (videoUUID: string) { + let webtorrentFilenames: string[] + let hlsFilenames: string[] + + for (const server of servers) { + const video = await server.videos.getWithToken({ id: videoUUID }) + + // Ensure we use the same filenames that the origin + + const localWebtorrentFilenames = video.files.map(f => basename(f.fileUrl)).sort() + const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort() + + if (webtorrentFilenames) expect(webtorrentFilenames).to.deep.equal(localWebtorrentFilenames) + else webtorrentFilenames = localWebtorrentFilenames + + if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames) + else hlsFilenames = localHLSFilenames + } + + return { webtorrentFilenames, hlsFilenames } +} + async function check1WebSeed (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid const webseeds = [ - `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` + `http://localhost:${servers[1].port}/static/webseed/` ] for (const server of servers) { @@ -107,40 +139,31 @@ async function check1WebSeed (videoUUID?: string) { const video = await server.videos.getWithToken({ id: videoUUID }) for (const f of video.files) { - checkMagnetWebseeds(f, webseeds, server) + await checkMagnetWebseeds(f, webseeds, server) } } + + await ensureSameFilenames(videoUUID) } async function check2Webseeds (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid const webseeds = [ - `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`, - `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` + `http://localhost:${servers[0].port}/static/redundancy/`, + `http://localhost:${servers[1].port}/static/webseed/` ] for (const server of servers) { const video = await server.videos.get({ id: videoUUID }) for (const file of video.files) { - checkMagnetWebseeds(file, webseeds, server) - - await makeGetRequest({ - url: servers[0].url, - expectedStatus: HttpStatusCode.OK_200, - path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) - await makeGetRequest({ - url: servers[1].url, - expectedStatus: HttpStatusCode.OK_200, - path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) + await checkMagnetWebseeds(file, webseeds, server) } } + const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) + const directories = [ 'test' + servers[0].internalServerNumber + '/redundancy', 'test' + servers[1].internalServerNumber + '/videos' @@ -150,14 +173,13 @@ async function check2Webseeds (videoUUID?: string) { const files = await readdir(join(root(), directory)) expect(files).to.have.length.at.least(4) - for (const resolution of [ 240, 360, 480, 720 ]) { - expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined - } + // Ensure we files exist on disk + expect(files.find(f => webtorrentFilenames.includes(f))).to.exist } } async function check0PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid for (const server of servers) { // With token to avoid issues with video follow constraints @@ -167,10 +189,12 @@ async function check0PlaylistRedundancies (videoUUID?: string) { expect(video.streamingPlaylists).to.have.lengthOf(1) expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) } + + await ensureSameFilenames(videoUUID) } async function check1PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid for (const server of servers) { const video = await server.videos.get({ id: videoUUID }) @@ -193,6 +217,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) { await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) } + const { hlsFilenames } = await ensureSameFilenames(videoUUID) + const directories = [ 'test' + servers[0].internalServerNumber + '/redundancy/hls', 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' @@ -202,11 +228,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) { const files = await readdir(join(root(), directory, videoUUID)) expect(files).to.have.length.at.least(4) - for (const resolution of [ 240, 360, 480, 720 ]) { - const filename = `${videoUUID}-${resolution}-fragmented.mp4` - - expect(files.find(f => f === filename)).to.not.be.undefined - } + // Ensure we files exist on disk + expect(files.find(f => hlsFilenames.includes(f))).to.exist } } @@ -322,7 +345,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos', join('playlists', 'hls') ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { @@ -372,7 +395,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { @@ -414,8 +437,8 @@ describe('Test videos redundancy', function () { it('Should view 2 times the first video to have > min_views config', async function () { this.timeout(80000) - await servers[0].videos.view({ id: video1Server2UUID }) - await servers[2].videos.view({ id: video1Server2UUID }) + await servers[0].videos.view({ id: video1Server2.uuid }) + await servers[2].videos.view({ id: video1Server2.uuid }) await wait(10000) await waitJobs(servers) @@ -436,12 +459,13 @@ describe('Test videos redundancy', function () { it('Should remove the video and the redundancy files', async function () { this.timeout(20000) - await servers[1].videos.remove({ id: video1Server2UUID }) + await saveVideoInServers(servers, video1Server2.uuid) + await servers[1].videos.remove({ id: video1Server2.uuid }) await waitJobs(servers) for (const server of servers) { - await checkVideoFilesWereRemoved(video1Server2UUID, server) + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) } }) @@ -482,8 +506,8 @@ describe('Test videos redundancy', function () { it('Should have 1 redundancy on the first video', async function () { this.timeout(160000) - await servers[0].videos.view({ id: video1Server2UUID }) - await servers[2].videos.view({ id: video1Server2UUID }) + await servers[0].videos.view({ id: video1Server2.uuid }) + await servers[2].videos.view({ id: video1Server2.uuid }) await wait(10000) await waitJobs(servers) @@ -499,12 +523,13 @@ describe('Test videos redundancy', function () { it('Should remove the video and the redundancy files', async function () { this.timeout(20000) - await servers[1].videos.remove({ id: video1Server2UUID }) + await saveVideoInServers(servers, video1Server2.uuid) + await servers[1].videos.remove({ id: video1Server2.uuid }) await waitJobs(servers) for (const server of servers) { - await checkVideoFilesWereRemoved(video1Server2UUID, server) + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) } }) @@ -527,7 +552,7 @@ describe('Test videos redundancy', function () { }) it('Should create a redundancy on first video', async function () { - await servers[0].redundancy.addVideo({ videoId: video1Server2Id }) + await servers[0].redundancy.addVideo({ videoId: video1Server2.id }) }) it('Should have 2 webseeds on the first video', async function () { @@ -562,7 +587,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { @@ -575,7 +600,7 @@ describe('Test videos redundancy', function () { async function checkContains (servers: PeerTubeServer[], str: string) { for (const server of servers) { - const video = await server.videos.get({ id: video1Server2UUID }) + const video = await server.videos.get({ id: video1Server2.uuid }) for (const f of video.files) { expect(f.magnetUri).to.contain(str) @@ -585,7 +610,7 @@ describe('Test videos redundancy', function () { async function checkNotContains (servers: PeerTubeServer[], str: string) { for (const server of servers) { - const video = await server.videos.get({ id: video1Server2UUID }) + const video = await server.videos.get({ id: video1Server2.uuid }) for (const f of video.files) { expect(f.magnetUri).to.not.contain(str) @@ -646,8 +671,8 @@ describe('Test videos redundancy', function () { await servers[0].servers.waitUntilLog('Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(video1Server2UUID) - await check1PlaylistRedundancies(video1Server2UUID) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } }) @@ -670,8 +695,8 @@ describe('Test videos redundancy', function () { await wait(1000) try { - await check1WebSeed(video1Server2UUID) - await check0PlaylistRedundancies(video1Server2UUID) + await check1WebSeed() + await check0PlaylistRedundancies() await check2Webseeds(video2Server2UUID) await check1PlaylistRedundancies(video2Server2UUID) @@ -700,7 +725,7 @@ describe('Test videos redundancy', function () { await waitJobs(servers) - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ join('redundancy', 'hls') ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts index 225145957..16372b039 100644 --- a/server/tests/api/users/users-multiple-servers.ts +++ b/server/tests/api/users/users-multiple-servers.ts @@ -10,18 +10,21 @@ import { createMultipleServers, doubleFollow, PeerTubeServer, + saveVideoInServers, setAccessTokensToServers, testImage, waitJobs } from '@shared/extra-utils' -import { User } from '@shared/models' +import { MyUser } from '@shared/models' const expect = chai.expect describe('Test users with multiple servers', function () { let servers: PeerTubeServer[] = [] - let user: User + + let user: MyUser let userId: number + let videoUUID: string let userAccessToken: string let userAvatarFilename: string @@ -45,18 +48,17 @@ describe('Test users with multiple servers', function () { await servers[0].videos.upload() { - const user = { - username: 'user1', - password: 'password' - } - const created = await servers[0].users.create(user) + const username = 'user1' + const created = await servers[0].users.create({ username }) userId = created.id - userAccessToken = await servers[0].login.getAccessToken(user) + userAccessToken = await servers[0].login.getAccessToken(username) } { const { uuid } = await servers[0].videos.upload({ token: userAccessToken }) videoUUID = uuid + + await saveVideoInServers(servers, videoUUID) } await waitJobs(servers) @@ -195,7 +197,7 @@ describe('Test users with multiple servers', function () { it('Should not have video files', async () => { for (const server of servers) { - await checkVideoFilesWereRemoved(videoUUID, server) + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) } }) diff --git a/server/tests/api/videos/audio-only.ts b/server/tests/api/videos/audio-only.ts index 15c3ae6d6..7fac6e738 100644 --- a/server/tests/api/videos/audio-only.ts +++ b/server/tests/api/videos/audio-only.ts @@ -2,7 +2,6 @@ import 'mocha' import * as chai from 'chai' -import { join } from 'path' import { getAudioStream, getVideoStreamSize } from '@server/helpers/ffprobe-utils' import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils' @@ -11,6 +10,8 @@ const expect = chai.expect describe('Test audio only video transcoding', function () { let servers: PeerTubeServer[] = [] let videoUUID: string + let webtorrentAudioFileUrl: string + let fragmentedAudioFileUrl: string before(async function () { this.timeout(120000) @@ -63,13 +64,18 @@ describe('Test audio only video transcoding', function () { expect(files[1].resolution.id).to.equal(240) expect(files[2].resolution.id).to.equal(0) } + + if (server.serverNumber === 1) { + webtorrentAudioFileUrl = video.files[2].fileUrl + fragmentedAudioFileUrl = video.streamingPlaylists[0].files[2].fileUrl + } } }) it('0p transcoded video should not have video', async function () { const paths = [ - servers[0].servers.buildDirectory(join('videos', videoUUID + '-0.mp4')), - servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', videoUUID, videoUUID + '-0-fragmented.mp4')) + servers[0].servers.buildWebTorrentFilePath(webtorrentAudioFileUrl), + servers[0].servers.buildFragmentedFilePath(videoUUID, fragmentedAudioFileUrl) ] for (const path of paths) { diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index d916abb09..f9220e4b3 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -13,6 +13,7 @@ import { dateIsValid, doubleFollow, PeerTubeServer, + saveVideoInServers, setAccessTokensToServers, testImage, wait, @@ -661,19 +662,19 @@ describe('Test multiple servers', function () { } }) - it('Should remove the videos 3 and 3-2 by asking server 3', async function () { - this.timeout(10000) + it('Should remove the videos 3 and 3-2 by asking server 3 and correctly delete files', async function () { + this.timeout(30000) - await servers[2].videos.remove({ id: toRemove[0].id }) - await servers[2].videos.remove({ id: toRemove[1].id }) + for (const id of [ toRemove[0].id, toRemove[1].id ]) { + await saveVideoInServers(servers, id) - await waitJobs(servers) - }) + await servers[2].videos.remove({ id }) - it('Should not have files of videos 3 and 3-2 on each server', async function () { - for (const server of servers) { - await checkVideoFilesWereRemoved(toRemove[0].uuid, server) - await checkVideoFilesWereRemoved(toRemove[1].uuid, server) + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) + } } }) diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts index a2d60eeec..c94d92cf2 100644 --- a/server/tests/api/videos/resumable-upload.ts +++ b/server/tests/api/videos/resumable-upload.ts @@ -99,8 +99,8 @@ describe('Test resumable upload', function () { this.timeout(30000) server = await createSingleServer(1) - await setAccessTokensToServers([server]) - await setDefaultVideoChannel([server]) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) const body = await server.users.getMyInfo() rootId = body.id @@ -170,13 +170,13 @@ describe('Test resumable upload', function () { const size = 1000 - const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}` + const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}` await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size }) await checkFileSize(uploadId, 0) }) }) after(async function () { - await cleanupTests([server]) + await cleanupTests([ server ]) }) }) diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index c0535be09..29dac6ec1 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -199,9 +199,10 @@ describe('Test a single server', function () { }) it('Should remove the video', async function () { + const video = await server.videos.get({ id: videoId }) await server.videos.remove({ id: videoId }) - await checkVideoFilesWereRemoved(videoUUID, server) + await checkVideoFilesWereRemoved({ video, server }) }) it('Should not have videos', async function () { diff --git a/server/tests/api/videos/video-captions.ts b/server/tests/api/videos/video-captions.ts index 4c8e28adf..3bb0d131c 100644 --- a/server/tests/api/videos/video-captions.ts +++ b/server/tests/api/videos/video-captions.ts @@ -178,9 +178,12 @@ describe('Test video captions', function () { }) it('Should remove the video, and thus all video captions', async function () { + const video = await servers[0].videos.get({ id: videoUUID }) + const { data: captions } = await servers[0].captions.list({ videoId: videoUUID }) + await servers[0].videos.remove({ id: videoUUID }) - await checkVideoFilesWereRemoved(videoUUID, servers[0]) + await checkVideoFilesWereRemoved({ server: servers[0], video, captions }) }) after(async function () { diff --git a/server/tests/api/videos/video-change-ownership.ts b/server/tests/api/videos/video-change-ownership.ts index 6ae6d3004..d6665fe4e 100644 --- a/server/tests/api/videos/video-change-ownership.ts +++ b/server/tests/api/videos/video-change-ownership.ts @@ -73,7 +73,7 @@ describe('Test video change ownership - nominal', function () { } const { id } = await servers[0].videos.upload({ token: firstUserToken, attributes }) - servers[0].store.video = await servers[0].videos.get({ id }) + servers[0].store.videoCreated = await servers[0].videos.get({ id }) } { @@ -109,7 +109,7 @@ describe('Test video change ownership - nominal', function () { it('Should send a request to change ownership of a video', async function () { this.timeout(15000) - await command.create({ token: firstUserToken, videoId: servers[0].store.video.id, username: secondUser }) + await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) }) it('Should only return a request to change ownership for the second user', async function () { @@ -135,7 +135,7 @@ describe('Test video change ownership - nominal', function () { it('Should accept the same change ownership request without crashing', async function () { this.timeout(10000) - await command.create({ token: firstUserToken, videoId: servers[0].store.video.id, username: secondUser }) + await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) }) it('Should not create multiple change ownership requests while one is waiting', async function () { @@ -163,7 +163,7 @@ describe('Test video change ownership - nominal', function () { it('Should send a new request to change ownership of a video', async function () { this.timeout(15000) - await command.create({ token: firstUserToken, videoId: servers[0].store.video.id, username: secondUser }) + await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) }) it('Should return two requests to change ownership for the second user', async function () { @@ -207,7 +207,7 @@ describe('Test video change ownership - nominal', function () { it('Should have the channel of the video updated', async function () { for (const server of servers) { - const video = await server.videos.get({ id: servers[0].store.video.uuid }) + const video = await server.videos.get({ id: servers[0].store.videoCreated.uuid }) expect(video.name).to.equal('my super name') expect(video.channel.displayName).to.equal('Main second channel') @@ -236,7 +236,7 @@ describe('Test video change ownership - nominal', function () { await waitJobs(servers) for (const server of servers) { - const video = await server.videos.get({ id: servers[0].store.video.uuid }) + const video = await server.videos.get({ id: servers[0].store.videoCreated.uuid }) expect(video.name).to.equal('my super name') expect(video.channel.displayName).to.equal('Main second channel') @@ -282,13 +282,13 @@ describe('Test video change ownership - quota too small', function () { const { data } = await server.videos.list() expect(data.length).to.equal(1) - server.store.video = data.find(video => video.name === 'my super name') + server.store.videoCreated = data.find(video => video.name === 'my super name') }) it('Should send a request to change ownership of a video', async function () { this.timeout(15000) - await server.changeOwnership.create({ token: firstUserToken, videoId: server.store.video.id, username: secondUser }) + await server.changeOwnership.create({ token: firstUserToken, videoId: server.store.videoCreated.id, username: secondUser }) }) it('Should only return a request to change ownership for the second user', async function () { diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts index 7845f7334..921d7ce64 100644 --- a/server/tests/api/videos/video-hls.ts +++ b/server/tests/api/videos/video-hls.ts @@ -19,6 +19,8 @@ import { } from '@shared/extra-utils' import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models' import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' +import { uuidRegex } from '@shared/core-utils' +import { basename } from 'path/posix' const expect = chai.expect @@ -38,14 +40,17 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0) else expect(videoDetails.files).to.have.lengthOf(resolutions.length) + // Check JSON files for (const resolution of resolutions) { const file = hlsFiles.find(f => f.resolution.id === resolution) expect(file).to.not.be.undefined expect(file.magnetUri).to.have.lengthOf.above(2) - expect(file.torrentUrl).to.equal(`http://${server.host}/lazy-static/torrents/${videoDetails.uuid}-${file.resolution.id}-hls.torrent`) - expect(file.fileUrl).to.equal( - `${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${videoDetails.uuid}-${file.resolution.id}-fragmented.mp4` + expect(file.torrentUrl).to.match( + new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) + ) + expect(file.fileUrl).to.match( + new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) ) expect(file.resolution.label).to.equal(resolution + 'p') @@ -58,6 +63,7 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h expect(torrent.files[0].path).to.exist.and.to.not.equal('') } + // Check master playlist { await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) @@ -69,13 +75,16 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h } } + // Check resolution playlists { for (const resolution of resolutions) { const subPlaylist = await server.streamingPlaylists.get({ url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8` }) - expect(subPlaylist).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`) + const file = hlsFiles.find(f => f.resolution.id === resolution) + expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) + expect(subPlaylist).to.contain(basename(file.fileUrl)) } } diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index e4892bb24..2a09e95bf 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -3,7 +3,6 @@ import 'mocha' import * as chai from 'chai' import { omit } from 'lodash' -import { join } from 'path' import { buildAbsoluteFixturePath, cleanupTests, @@ -11,6 +10,7 @@ import { doubleFollow, generateHighBitrateVideo, generateVideoWithFramerate, + getFileSize, makeGetRequest, PeerTubeServer, setAccessTokensToServers, @@ -271,7 +271,8 @@ describe('Test video transcoding', function () { expect(videoDetails.files).to.have.lengthOf(4) - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-240.mp4')) + const file = videoDetails.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const probe = await getAudioStream(path) if (probe.audioStream) { @@ -300,8 +301,9 @@ describe('Test video transcoding', function () { const video = data.find(v => v.name === attributes.name) const videoDetails = await server.videos.get({ id: video.id }) - expect(videoDetails.files).to.have.lengthOf(4) - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-240.mp4')) + const file = videoDetails.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) + const probe = await getAudioStream(path) expect(probe).to.not.have.property('audioStream') } @@ -328,7 +330,9 @@ describe('Test video transcoding', function () { const fixturePath = buildAbsoluteFixturePath(attributes.fixture) const fixtureVideoProbe = await getAudioStream(fixturePath) - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-240.mp4')) + + const file = videoDetails.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const videoProbe = await getAudioStream(path) @@ -485,14 +489,16 @@ describe('Test video transcoding', function () { expect(videoDetails.files[2].fps).to.be.below(31) expect(videoDetails.files[3].fps).to.be.below(31) - for (const resolution of [ '240', '360', '480' ]) { - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-' + resolution + '.mp4')) + for (const resolution of [ 240, 360, 480 ]) { + const file = videoDetails.files.find(f => f.resolution.id === resolution) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const fps = await getVideoFileFPS(path) expect(fps).to.be.below(31) } - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-720.mp4')) + const file = videoDetails.files.find(f => f.resolution.id === 720) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const fps = await getVideoFileFPS(path) expect(fps).to.be.above(58).and.below(62) @@ -524,16 +530,19 @@ describe('Test video transcoding', function () { for (const server of servers) { const { data } = await server.videos.list() - const video = data.find(v => v.name === attributes.name) + const { id } = data.find(v => v.name === attributes.name) + const video = await server.videos.get({ id }) { - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-240.mp4')) + const file = video.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const fps = await getVideoFileFPS(path) expect(fps).to.be.equal(25) } { - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-720.mp4')) + const file = video.files.find(f => f.resolution.id === 720) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const fps = await getVideoFileFPS(path) expect(fps).to.be.equal(59) } @@ -542,6 +551,7 @@ describe('Test video transcoding', function () { }) describe('Bitrate control', function () { + it('Should respect maximum bitrate values', async function () { this.timeout(160_000) @@ -567,17 +577,19 @@ describe('Test video transcoding', function () { for (const server of servers) { const { data } = await server.videos.list() - const video = data.find(v => v.name === attributes.name) + const { id } = data.find(v => v.name === attributes.name) + const video = await server.videos.get({ id }) - for (const resolution of [ '240', '360', '480', '720', '1080' ]) { - const path = servers[1].servers.buildDirectory(join('videos', video.uuid + '-' + resolution + '.mp4')) + for (const resolution of [ 240, 360, 480, 720, 1080 ]) { + const file = video.files.find(f => f.resolution.id === resolution) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const bitrate = await getVideoFileBitrate(path) const fps = await getVideoFileFPS(path) - const resolution2 = await getVideoFileResolution(path) + const { videoFileResolution } = await getVideoFileResolution(path) - expect(resolution2.videoFileResolution.toString()).to.equal(resolution) - expect(bitrate).to.be.below(getMaxBitrate(resolution2.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) + expect(videoFileResolution).to.equal(resolution) + expect(bitrate).to.be.below(getMaxBitrate(videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) } } }) @@ -608,14 +620,18 @@ describe('Test video transcoding', function () { fixture: 'low-bitrate.mp4' } - const { uuid } = await servers[1].videos.upload({ attributes }) + const { id } = await servers[1].videos.upload({ attributes }) await waitJobs(servers) + const video = await servers[1].videos.get({ id }) + const resolutions = [ 240, 360, 480, 720, 1080 ] for (const r of resolutions) { - const path = `videos/${uuid}-${r}.mp4` - const size = await servers[1].servers.getServerFileSize(path) + const file = video.files.find(f => f.resolution.id === r) + + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) + const size = await getFileSize(path) expect(size, `${path} not below ${60_000}`).to.be.below(60_000) } }) @@ -630,7 +646,9 @@ describe('Test video transcoding', function () { await waitJobs(servers) { - const path = servers[1].servers.buildDirectory(join('videos', videoUUID + '-240.mp4')) + const video = await servers[1].videos.get({ id: videoUUID }) + const file = video.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebTorrentFilePath(file.fileUrl) const metadata = await getMetadataFromFile(path) // expected format properties diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts index 242994273..5d16b28a4 100644 --- a/server/tests/plugins/plugin-helpers.ts +++ b/server/tests/plugins/plugin-helpers.ts @@ -232,7 +232,7 @@ describe('Test plugin helpers', function () { this.timeout(40000) // Should not throw -> video exists - await servers[0].videos.get({ id: videoUUID }) + const video = await servers[0].videos.get({ id: videoUUID }) // Should delete the video await servers[0].videos.view({ id: videoUUID }) @@ -246,7 +246,7 @@ describe('Test plugin helpers', function () { if (err.message.includes('exists')) throw err } - await checkVideoFilesWereRemoved(videoUUID, servers[0]) + await checkVideoFilesWereRemoved({ server: servers[0], video }) }) it('Should have fetched the video by URL', async function () { diff --git a/shared/core-utils/miscs/index.ts b/shared/core-utils/miscs/index.ts index afd147f24..7764e69ad 100644 --- a/shared/core-utils/miscs/index.ts +++ b/shared/core-utils/miscs/index.ts @@ -1,3 +1,4 @@ export * from './date' export * from './miscs' export * from './types' +export * from './regexp' diff --git a/shared/core-utils/miscs/regexp.ts b/shared/core-utils/miscs/regexp.ts new file mode 100644 index 000000000..862b8e00f --- /dev/null +++ b/shared/core-utils/miscs/regexp.ts @@ -0,0 +1 @@ +export const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 5bdcbac52..d37a7f39c 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts @@ -3,7 +3,7 @@ import { copy } from 'fs-extra' import { join } from 'path' import { root } from '@server/helpers/core-utils' import { randomInt } from '../../core-utils/miscs/miscs' -import { VideoChannel } from '../../models/videos' +import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '../../models/videos' import { BulkCommand } from '../bulk' import { CLICommand } from '../cli' import { CustomPagesCommand } from '../custom-pages' @@ -75,19 +75,9 @@ export class PeerTubeServer { channel?: VideoChannel - video?: { - id: number - uuid: string - shortUUID: string - name?: string - url?: string - - account?: { - name: string - } - - embedPath?: string - } + video?: Video + videoCreated?: VideoCreateResult + videoDetails?: VideoDetails videos?: { id: number, uuid: string }[] } diff --git a/shared/extra-utils/server/servers-command.ts b/shared/extra-utils/server/servers-command.ts index a78921f2a..441c728c1 100644 --- a/shared/extra-utils/server/servers-command.ts +++ b/shared/extra-utils/server/servers-command.ts @@ -1,6 +1,7 @@ import { exec } from 'child_process' import { copy, ensureDir, readFile, remove } from 'fs-extra' import { join } from 'path' +import { basename } from 'path/posix' import { root } from '@server/helpers/core-utils' import { HttpStatusCode } from '@shared/models' import { getFileSize, isGithubCI, wait } from '../miscs' @@ -72,6 +73,14 @@ export class ServersCommand extends AbstractCommand { return join(root(), 'test' + this.server.internalServerNumber, directory) } + buildWebTorrentFilePath (fileUrl: string) { + return this.buildDirectory(join('videos', basename(fileUrl))) + } + + buildFragmentedFilePath (videoUUID: string, fileUrl: string) { + return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) + } + async getServerFileSize (subPath: string) { const path = this.server.servers.buildDirectory(subPath) diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index 1ae3fefc1..db40c27be 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts @@ -1,4 +1,5 @@ import { expect } from 'chai' +import { basename } from 'path' import { sha256 } from '@server/helpers/core-utils' import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models' import { PeerTubeServer } from '../server' @@ -16,7 +17,8 @@ async function checkSegmentHash (options: { const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8` }) - const videoName = `${videoUUID}-${resolution}-fragmented.mp4` + const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) + const videoName = basename(file.fileUrl) const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 9a9bfb3cf..a1d2ba0fc 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -2,9 +2,10 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' -import { join } from 'path' +import { basename, join } from 'path' import { getLowercaseExtension } from '@server/helpers/core-utils' -import { HttpStatusCode } from '@shared/models' +import { uuidRegex } from '@shared/core-utils' +import { HttpStatusCode, VideoCaption, VideoDetails } from '@shared/models' import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' import { dateIsValid, testImage, webtorrentAdd } from '../miscs' import { makeRawRequest } from '../requests/requests' @@ -12,33 +13,66 @@ import { waitJobs } from '../server' import { PeerTubeServer } from '../server/server' import { VideoEdit } from './videos-command' -async function checkVideoFilesWereRemoved ( - videoUUID: string, - server: PeerTubeServer, - directories = [ - 'redundancy', - 'videos', - 'thumbnails', - 'torrents', - 'previews', - 'captions', - join('playlists', 'hls'), - join('redundancy', 'hls') - ] -) { - for (const directory of directories) { +async function checkVideoFilesWereRemoved (options: { + server: PeerTubeServer + video: VideoDetails + captions?: VideoCaption[] + onlyVideoFiles?: boolean // default false +}) { + const { video, server, captions = [], onlyVideoFiles = false } = options + + const webtorrentFiles = video.files || [] + const hlsFiles = video.streamingPlaylists[0]?.files || [] + + const thumbnailName = basename(video.thumbnailPath) + const previewName = basename(video.previewPath) + + const torrentNames = webtorrentFiles.concat(hlsFiles).map(f => basename(f.torrentUrl)) + + const captionNames = captions.map(c => basename(c.captionPath)) + + const webtorrentFilenames = webtorrentFiles.map(f => basename(f.fileUrl)) + const hlsFilenames = hlsFiles.map(f => basename(f.fileUrl)) + + let directories: { [ directory: string ]: string[] } = { + videos: webtorrentFilenames, + redundancy: webtorrentFilenames, + [join('playlists', 'hls')]: hlsFilenames, + [join('redundancy', 'hls')]: hlsFilenames + } + + if (onlyVideoFiles !== true) { + directories = { + ...directories, + + thumbnails: [ thumbnailName ], + previews: [ previewName ], + torrents: torrentNames, + captions: captionNames + } + } + + for (const directory of Object.keys(directories)) { const directoryPath = server.servers.buildDirectory(directory) const directoryExists = await pathExists(directoryPath) if (directoryExists === false) continue - const files = await readdir(directoryPath) - for (const file of files) { - expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID) + const existingFiles = await readdir(directoryPath) + for (const existingFile of existingFiles) { + for (const shouldNotExist of directories[directory]) { + expect(existingFile, `File ${existingFile} should not exist in ${directoryPath}`).to.not.contain(shouldNotExist) + } } } } +async function saveVideoInServers (servers: PeerTubeServer[], uuid: string) { + for (const server of servers) { + server.store.videoDetails = await server.videos.get({ id: uuid }) + } +} + function checkUploadVideoParam ( server: PeerTubeServer, token: string, @@ -156,18 +190,16 @@ async function completeVideoCheck ( expect(file.magnetUri).to.have.lengthOf.above(2) - expect(file.torrentDownloadUrl).to.equal(`http://${host}/download/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`) - expect(file.torrentUrl).to.equal(`http://${host}/lazy-static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`) + expect(file.torrentDownloadUrl).to.match(new RegExp(`http://${host}/download/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) + expect(file.torrentUrl).to.match(new RegExp(`http://${host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) - expect(file.fileUrl).to.equal(`http://${originHost}/static/webseed/${videoDetails.uuid}-${file.resolution.id}${extension}`) - expect(file.fileDownloadUrl).to.equal(`http://${originHost}/download/videos/${videoDetails.uuid}-${file.resolution.id}${extension}`) + expect(file.fileUrl).to.match(new RegExp(`http://${originHost}/static/webseed/${uuidRegex}-${file.resolution.id}${extension}`)) + expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) await Promise.all([ makeRawRequest(file.torrentUrl, 200), makeRawRequest(file.torrentDownloadUrl, 200), - makeRawRequest(file.metadataUrl, 200), - // Backward compatibility - makeRawRequest(`http://${originHost}/static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`, 200) + makeRawRequest(file.metadataUrl, 200) ]) expect(file.resolution.id).to.equal(attributeFile.resolution) @@ -215,5 +247,6 @@ export { checkUploadVideoParam, completeVideoCheck, uploadRandomVideoOnServers, - checkVideoFilesWereRemoved + checkVideoFilesWereRemoved, + saveVideoInServers } -- 2.41.0