X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Ftests%2Fapi%2Fredundancy%2Fredundancy.ts;h=5abed358f99737ee3f885f25ea273f09a63a56f5;hb=8a6828b1664ce3fc535d23c54ed22bab35588d06;hp=a6559d304f775006807a0baa162522e37f416ffb;hpb=4c7e60bc17ee5830399bac4aa273356903421b4c;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index a6559d304..5abed358f 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -1,51 +1,56 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import 'mocha' -import * as chai from 'chai' +import { expect } from 'chai' import { readdir } from 'fs-extra' -import * as magnetUtil from 'magnet-uri' -import { join } from 'path' +import magnetUtil from 'magnet-uri' +import { basename, join } from 'path' +import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared' +import { root, wait } from '@shared/core-utils' +import { + HttpStatusCode, + VideoDetails, + VideoFile, + VideoPrivacy, + VideoRedundancyStrategy, + VideoRedundancyStrategyWithManual +} from '@shared/models' import { - checkSegmentHash, - checkVideoFilesWereRemoved, cleanupTests, createMultipleServers, doubleFollow, killallServers, - makeGetRequest, + makeRawRequest, PeerTubeServer, - root, setAccessTokensToServers, - wait, waitJobs -} from '@shared/extra-utils' -import { HttpStatusCode, VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models' - -const expect = chai.expect +} from '@shared/server-commands' let servers: PeerTubeServer[] = [] -let video1Server2UUID: string -let video1Server2Id: number +let video1Server2: VideoDetails -function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: PeerTubeServer) { +async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) { const parsed = magnetUtil.decode(file.magnetUri) for (const ws of baseWebseeds) { - const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) + const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`) expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined } expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) + + for (const url of parsed.urlList) { + await makeRawRequest(url, HttpStatusCode.OK_200) + } } -async function createSingleServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) { +async function createServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) { const strategies: any[] = [] if (strategy !== null) { strategies.push( { min_lifetime: '1 hour', - strategy: strategy, + strategy, size: '400KB', ...additionalParams @@ -76,11 +81,10 @@ async function createSingleServers (strategy: VideoRedundancyStrategy | null, ad await setAccessTokensToServers(servers) { - const { uuid, id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) - video1Server2UUID = uuid - video1Server2Id = id + const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) + video1Server2 = await servers[1].videos.get({ id }) - await servers[1].videos.view({ id: video1Server2UUID }) + await servers[1].views.simulateView({ id }) } await waitJobs(servers) @@ -95,11 +99,33 @@ async function createSingleServers (strategy: VideoRedundancyStrategy | null, ad await waitJobs(servers) } +async function ensureSameFilenames (videoUUID: string) { + let webtorrentFilenames: string[] + let hlsFilenames: string[] + + for (const server of servers) { + const video = await server.videos.getWithToken({ id: videoUUID }) + + // Ensure we use the same filenames that the origin + + const localWebtorrentFilenames = video.files.map(f => basename(f.fileUrl)).sort() + const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort() + + if (webtorrentFilenames) expect(webtorrentFilenames).to.deep.equal(localWebtorrentFilenames) + else webtorrentFilenames = localWebtorrentFilenames + + if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames) + else hlsFilenames = localHLSFilenames + } + + return { webtorrentFilenames, hlsFilenames } +} + async function check1WebSeed (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid const webseeds = [ - `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` + `http://localhost:${servers[1].port}/static/webseed/` ] for (const server of servers) { @@ -107,40 +133,31 @@ async function check1WebSeed (videoUUID?: string) { const video = await server.videos.getWithToken({ id: videoUUID }) for (const f of video.files) { - checkMagnetWebseeds(f, webseeds, server) + await checkMagnetWebseeds(f, webseeds, server) } } + + await ensureSameFilenames(videoUUID) } async function check2Webseeds (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid const webseeds = [ - `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`, - `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` + `http://localhost:${servers[0].port}/static/redundancy/`, + `http://localhost:${servers[1].port}/static/webseed/` ] for (const server of servers) { const video = await server.videos.get({ id: videoUUID }) for (const file of video.files) { - checkMagnetWebseeds(file, webseeds, server) - - await makeGetRequest({ - url: servers[0].url, - expectedStatus: HttpStatusCode.OK_200, - path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) - await makeGetRequest({ - url: servers[1].url, - expectedStatus: HttpStatusCode.OK_200, - path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) + await checkMagnetWebseeds(file, webseeds, server) } } + const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) + const directories = [ 'test' + servers[0].internalServerNumber + '/redundancy', 'test' + servers[1].internalServerNumber + '/videos' @@ -150,14 +167,13 @@ async function check2Webseeds (videoUUID?: string) { const files = await readdir(join(root(), directory)) expect(files).to.have.length.at.least(4) - for (const resolution of [ 240, 360, 480, 720 ]) { - expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined - } + // Ensure we files exist on disk + expect(files.find(f => webtorrentFilenames.includes(f))).to.exist } } async function check0PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid for (const server of servers) { // With token to avoid issues with video follow constraints @@ -167,10 +183,12 @@ async function check0PlaylistRedundancies (videoUUID?: string) { expect(video.streamingPlaylists).to.have.lengthOf(1) expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) } + + await ensureSameFilenames(videoUUID) } async function check1PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2UUID + if (!videoUUID) videoUUID = video1Server2.uuid for (const server of servers) { const video = await server.videos.get({ id: videoUUID }) @@ -183,16 +201,18 @@ async function check1PlaylistRedundancies (videoUUID?: string) { expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) } - const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' - const baseUrlSegment = servers[0].url + '/static/redundancy/hls' + const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID + const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID const video = await servers[0].videos.get({ id: videoUUID }) const hlsPlaylist = video.streamingPlaylists[0] for (const resolution of [ 240, 360, 480, 720 ]) { - await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) + await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist }) } + const { hlsFilenames } = await ensureSameFilenames(videoUUID) + const directories = [ 'test' + servers[0].internalServerNumber + '/redundancy/hls', 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' @@ -202,11 +222,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) { const files = await readdir(join(root(), directory, videoUUID)) expect(files).to.have.length.at.least(4) - for (const resolution of [ 240, 360, 480, 720 ]) { - const filename = `${videoUUID}-${resolution}-fragmented.mp4` - - expect(files.find(f => f === filename)).to.not.be.undefined - } + // Ensure we files exist on disk + expect(files.find(f => hlsFilenames.includes(f))).to.exist } } @@ -284,9 +301,9 @@ describe('Test videos redundancy', function () { const strategy = 'most-views' before(function () { - this.timeout(120000) + this.timeout(240000) - return createSingleServers(strategy) + return createServers(strategy) }) it('Should have 1 webseed on the first video', async function () { @@ -322,7 +339,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos', join('playlists', 'hls') ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { @@ -334,9 +351,9 @@ describe('Test videos redundancy', function () { const strategy = 'trending' before(function () { - this.timeout(120000) + this.timeout(240000) - return createSingleServers(strategy) + return createServers(strategy) }) it('Should have 1 webseed on the first video', async function () { @@ -361,7 +378,20 @@ describe('Test videos redundancy', function () { await checkStatsWith1Redundancy(strategy) }) - it('Should unfollow on server 1 and remove duplicated videos', async function () { + it('Should unfollow server 3 and keep duplicated videos', async function () { + this.timeout(80000) + + await servers[0].follows.unfollow({ target: servers[2] }) + + await waitJobs(servers) + await wait(5000) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + }) + + it('Should unfollow server 2 and remove duplicated videos', async function () { this.timeout(80000) await servers[0].follows.unfollow({ target: servers[1] }) @@ -372,7 +402,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { @@ -384,9 +414,9 @@ describe('Test videos redundancy', function () { const strategy = 'recently-added' before(function () { - this.timeout(120000) + this.timeout(240000) - return createSingleServers(strategy, { min_views: 3 }) + return createServers(strategy, { min_views: 3 }) }) it('Should have 1 webseed on the first video', async function () { @@ -414,8 +444,8 @@ describe('Test videos redundancy', function () { it('Should view 2 times the first video to have > min_views config', async function () { this.timeout(80000) - await servers[0].videos.view({ id: video1Server2UUID }) - await servers[2].videos.view({ id: video1Server2UUID }) + await servers[0].views.simulateView({ id: video1Server2.uuid }) + await servers[2].views.simulateView({ id: video1Server2.uuid }) await wait(10000) await waitJobs(servers) @@ -436,12 +466,13 @@ describe('Test videos redundancy', function () { it('Should remove the video and the redundancy files', async function () { this.timeout(20000) - await servers[1].videos.remove({ id: video1Server2UUID }) + await saveVideoInServers(servers, video1Server2.uuid) + await servers[1].videos.remove({ id: video1Server2.uuid }) await waitJobs(servers) for (const server of servers) { - await checkVideoFilesWereRemoved(video1Server2UUID, server) + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) } }) @@ -454,9 +485,9 @@ describe('Test videos redundancy', function () { const strategy = 'recently-added' before(async function () { - this.timeout(120000) + this.timeout(240000) - await createSingleServers(strategy, { min_views: 3 }, false) + await createServers(strategy, { min_views: 3 }, false) }) it('Should have 0 playlist redundancy on the first video', async function () { @@ -482,8 +513,8 @@ describe('Test videos redundancy', function () { it('Should have 1 redundancy on the first video', async function () { this.timeout(160000) - await servers[0].videos.view({ id: video1Server2UUID }) - await servers[2].videos.view({ id: video1Server2UUID }) + await servers[0].views.simulateView({ id: video1Server2.uuid }) + await servers[2].views.simulateView({ id: video1Server2.uuid }) await wait(10000) await waitJobs(servers) @@ -499,12 +530,13 @@ describe('Test videos redundancy', function () { it('Should remove the video and the redundancy files', async function () { this.timeout(20000) - await servers[1].videos.remove({ id: video1Server2UUID }) + await saveVideoInServers(servers, video1Server2.uuid) + await servers[1].videos.remove({ id: video1Server2.uuid }) await waitJobs(servers) for (const server of servers) { - await checkVideoFilesWereRemoved(video1Server2UUID, server) + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) } }) @@ -515,9 +547,9 @@ describe('Test videos redundancy', function () { describe('With manual strategy', function () { before(function () { - this.timeout(120000) + this.timeout(240000) - return createSingleServers(null) + return createServers(null) }) it('Should have 1 webseed on the first video', async function () { @@ -527,7 +559,7 @@ describe('Test videos redundancy', function () { }) it('Should create a redundancy on first video', async function () { - await servers[0].redundancy.addVideo({ videoId: video1Server2Id }) + await servers[0].redundancy.addVideo({ videoId: video1Server2.id }) }) it('Should have 2 webseeds on the first video', async function () { @@ -562,7 +594,7 @@ describe('Test videos redundancy', function () { await check1WebSeed() await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () { @@ -575,7 +607,7 @@ describe('Test videos redundancy', function () { async function checkContains (servers: PeerTubeServer[], str: string) { for (const server of servers) { - const video = await server.videos.get({ id: video1Server2UUID }) + const video = await server.videos.get({ id: video1Server2.uuid }) for (const f of video.files) { expect(f.magnetUri).to.contain(str) @@ -585,7 +617,7 @@ describe('Test videos redundancy', function () { async function checkNotContains (servers: PeerTubeServer[], str: string) { for (const server of servers) { - const video = await server.videos.get({ id: video1Server2UUID }) + const video = await server.videos.get({ id: video1Server2.uuid }) for (const f of video.files) { expect(f.magnetUri).to.not.contain(str) @@ -594,9 +626,9 @@ describe('Test videos redundancy', function () { } before(async function () { - this.timeout(120000) + this.timeout(240000) - await createSingleServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) await enableRedundancyOnServer1() }) @@ -636,9 +668,9 @@ describe('Test videos redundancy', function () { const strategy = 'recently-added' before(async function () { - this.timeout(120000) + this.timeout(240000) - await createSingleServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) await enableRedundancyOnServer1() @@ -646,8 +678,8 @@ describe('Test videos redundancy', function () { await servers[0].servers.waitUntilLog('Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(video1Server2UUID) - await check1PlaylistRedundancies(video1Server2UUID) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith1Redundancy(strategy) const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } }) @@ -660,7 +692,7 @@ describe('Test videos redundancy', function () { }) it('Should cache video 2 webseeds on the first video', async function () { - this.timeout(120000) + this.timeout(240000) await waitJobs(servers) @@ -670,8 +702,8 @@ describe('Test videos redundancy', function () { await wait(1000) try { - await check1WebSeed(video1Server2UUID) - await check0PlaylistRedundancies(video1Server2UUID) + await check1WebSeed() + await check0PlaylistRedundancies() await check2Webseeds(video2Server2UUID) await check1PlaylistRedundancies(video2Server2UUID) @@ -700,7 +732,7 @@ describe('Test videos redundancy', function () { await waitJobs(servers) - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ join('redundancy', 'hls') ]) + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) }) after(async function () {