aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/tests/api/redundancy/redundancy.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/tests/api/redundancy/redundancy.ts')
-rw-r--r--server/tests/api/redundancy/redundancy.ts145
1 files changed, 85 insertions, 60 deletions
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index a6559d304..e1a12f5f8 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -4,7 +4,7 @@ import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { readdir } from 'fs-extra' 5import { readdir } from 'fs-extra'
6import * as magnetUtil from 'magnet-uri' 6import * as magnetUtil from 'magnet-uri'
7import { join } from 'path' 7import { basename, join } from 'path'
8import { 8import {
9 checkSegmentHash, 9 checkSegmentHash,
10 checkVideoFilesWereRemoved, 10 checkVideoFilesWereRemoved,
@@ -12,30 +12,41 @@ import {
12 createMultipleServers, 12 createMultipleServers,
13 doubleFollow, 13 doubleFollow,
14 killallServers, 14 killallServers,
15 makeGetRequest, 15 makeRawRequest,
16 PeerTubeServer, 16 PeerTubeServer,
17 root, 17 root,
18 saveVideoInServers,
18 setAccessTokensToServers, 19 setAccessTokensToServers,
19 wait, 20 wait,
20 waitJobs 21 waitJobs
21} from '@shared/extra-utils' 22} from '@shared/extra-utils'
22import { HttpStatusCode, VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models' 23import {
24 HttpStatusCode,
25 VideoDetails,
26 VideoFile,
27 VideoPrivacy,
28 VideoRedundancyStrategy,
29 VideoRedundancyStrategyWithManual
30} from '@shared/models'
23 31
24const expect = chai.expect 32const expect = chai.expect
25 33
26let servers: PeerTubeServer[] = [] 34let servers: PeerTubeServer[] = []
27let video1Server2UUID: string 35let video1Server2: VideoDetails
28let video1Server2Id: number
29 36
30function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: PeerTubeServer) { 37async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) {
31 const parsed = magnetUtil.decode(file.magnetUri) 38 const parsed = magnetUtil.decode(file.magnetUri)
32 39
33 for (const ws of baseWebseeds) { 40 for (const ws of baseWebseeds) {
34 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) 41 const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`)
35 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined 42 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
36 } 43 }
37 44
38 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) 45 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
46
47 for (const url of parsed.urlList) {
48 await makeRawRequest(url, HttpStatusCode.OK_200)
49 }
39} 50}
40 51
41async function createSingleServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) { 52async function createSingleServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
@@ -76,11 +87,10 @@ async function createSingleServers (strategy: VideoRedundancyStrategy | null, ad
76 await setAccessTokensToServers(servers) 87 await setAccessTokensToServers(servers)
77 88
78 { 89 {
79 const { uuid, id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) 90 const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
80 video1Server2UUID = uuid 91 video1Server2 = await servers[1].videos.get({ id })
81 video1Server2Id = id
82 92
83 await servers[1].videos.view({ id: video1Server2UUID }) 93 await servers[1].videos.view({ id })
84 } 94 }
85 95
86 await waitJobs(servers) 96 await waitJobs(servers)
@@ -95,11 +105,33 @@ async function createSingleServers (strategy: VideoRedundancyStrategy | null, ad
95 await waitJobs(servers) 105 await waitJobs(servers)
96} 106}
97 107
108async function ensureSameFilenames (videoUUID: string) {
109 let webtorrentFilenames: string[]
110 let hlsFilenames: string[]
111
112 for (const server of servers) {
113 const video = await server.videos.getWithToken({ id: videoUUID })
114
115 // Ensure we use the same filenames that the origin
116
117 const localWebtorrentFilenames = video.files.map(f => basename(f.fileUrl)).sort()
118 const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort()
119
120 if (webtorrentFilenames) expect(webtorrentFilenames).to.deep.equal(localWebtorrentFilenames)
121 else webtorrentFilenames = localWebtorrentFilenames
122
123 if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames)
124 else hlsFilenames = localHLSFilenames
125 }
126
127 return { webtorrentFilenames, hlsFilenames }
128}
129
98async function check1WebSeed (videoUUID?: string) { 130async function check1WebSeed (videoUUID?: string) {
99 if (!videoUUID) videoUUID = video1Server2UUID 131 if (!videoUUID) videoUUID = video1Server2.uuid
100 132
101 const webseeds = [ 133 const webseeds = [
102 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` 134 `http://localhost:${servers[1].port}/static/webseed/`
103 ] 135 ]
104 136
105 for (const server of servers) { 137 for (const server of servers) {
@@ -107,40 +139,31 @@ async function check1WebSeed (videoUUID?: string) {
107 const video = await server.videos.getWithToken({ id: videoUUID }) 139 const video = await server.videos.getWithToken({ id: videoUUID })
108 140
109 for (const f of video.files) { 141 for (const f of video.files) {
110 checkMagnetWebseeds(f, webseeds, server) 142 await checkMagnetWebseeds(f, webseeds, server)
111 } 143 }
112 } 144 }
145
146 await ensureSameFilenames(videoUUID)
113} 147}
114 148
115async function check2Webseeds (videoUUID?: string) { 149async function check2Webseeds (videoUUID?: string) {
116 if (!videoUUID) videoUUID = video1Server2UUID 150 if (!videoUUID) videoUUID = video1Server2.uuid
117 151
118 const webseeds = [ 152 const webseeds = [
119 `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`, 153 `http://localhost:${servers[0].port}/static/redundancy/`,
120 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` 154 `http://localhost:${servers[1].port}/static/webseed/`
121 ] 155 ]
122 156
123 for (const server of servers) { 157 for (const server of servers) {
124 const video = await server.videos.get({ id: videoUUID }) 158 const video = await server.videos.get({ id: videoUUID })
125 159
126 for (const file of video.files) { 160 for (const file of video.files) {
127 checkMagnetWebseeds(file, webseeds, server) 161 await checkMagnetWebseeds(file, webseeds, server)
128
129 await makeGetRequest({
130 url: servers[0].url,
131 expectedStatus: HttpStatusCode.OK_200,
132 path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
133 contentType: null
134 })
135 await makeGetRequest({
136 url: servers[1].url,
137 expectedStatus: HttpStatusCode.OK_200,
138 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
139 contentType: null
140 })
141 } 162 }
142 } 163 }
143 164
165 const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
166
144 const directories = [ 167 const directories = [
145 'test' + servers[0].internalServerNumber + '/redundancy', 168 'test' + servers[0].internalServerNumber + '/redundancy',
146 'test' + servers[1].internalServerNumber + '/videos' 169 'test' + servers[1].internalServerNumber + '/videos'
@@ -150,14 +173,13 @@ async function check2Webseeds (videoUUID?: string) {
150 const files = await readdir(join(root(), directory)) 173 const files = await readdir(join(root(), directory))
151 expect(files).to.have.length.at.least(4) 174 expect(files).to.have.length.at.least(4)
152 175
153 for (const resolution of [ 240, 360, 480, 720 ]) { 176 // Ensure we files exist on disk
154 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined 177 expect(files.find(f => webtorrentFilenames.includes(f))).to.exist
155 }
156 } 178 }
157} 179}
158 180
159async function check0PlaylistRedundancies (videoUUID?: string) { 181async function check0PlaylistRedundancies (videoUUID?: string) {
160 if (!videoUUID) videoUUID = video1Server2UUID 182 if (!videoUUID) videoUUID = video1Server2.uuid
161 183
162 for (const server of servers) { 184 for (const server of servers) {
163 // With token to avoid issues with video follow constraints 185 // With token to avoid issues with video follow constraints
@@ -167,10 +189,12 @@ async function check0PlaylistRedundancies (videoUUID?: string) {
167 expect(video.streamingPlaylists).to.have.lengthOf(1) 189 expect(video.streamingPlaylists).to.have.lengthOf(1)
168 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) 190 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
169 } 191 }
192
193 await ensureSameFilenames(videoUUID)
170} 194}
171 195
172async function check1PlaylistRedundancies (videoUUID?: string) { 196async function check1PlaylistRedundancies (videoUUID?: string) {
173 if (!videoUUID) videoUUID = video1Server2UUID 197 if (!videoUUID) videoUUID = video1Server2.uuid
174 198
175 for (const server of servers) { 199 for (const server of servers) {
176 const video = await server.videos.get({ id: videoUUID }) 200 const video = await server.videos.get({ id: videoUUID })
@@ -193,6 +217,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
193 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) 217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist })
194 } 218 }
195 219
220 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
221
196 const directories = [ 222 const directories = [
197 'test' + servers[0].internalServerNumber + '/redundancy/hls', 223 'test' + servers[0].internalServerNumber + '/redundancy/hls',
198 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' 224 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
@@ -202,11 +228,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
202 const files = await readdir(join(root(), directory, videoUUID)) 228 const files = await readdir(join(root(), directory, videoUUID))
203 expect(files).to.have.length.at.least(4) 229 expect(files).to.have.length.at.least(4)
204 230
205 for (const resolution of [ 240, 360, 480, 720 ]) { 231 // Ensure we files exist on disk
206 const filename = `${videoUUID}-${resolution}-fragmented.mp4` 232 expect(files.find(f => hlsFilenames.includes(f))).to.exist
207
208 expect(files.find(f => f === filename)).to.not.be.undefined
209 }
210 } 233 }
211} 234}
212 235
@@ -322,7 +345,7 @@ describe('Test videos redundancy', function () {
322 await check1WebSeed() 345 await check1WebSeed()
323 await check0PlaylistRedundancies() 346 await check0PlaylistRedundancies()
324 347
325 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos', join('playlists', 'hls') ]) 348 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
326 }) 349 })
327 350
328 after(async function () { 351 after(async function () {
@@ -372,7 +395,7 @@ describe('Test videos redundancy', function () {
372 await check1WebSeed() 395 await check1WebSeed()
373 await check0PlaylistRedundancies() 396 await check0PlaylistRedundancies()
374 397
375 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ]) 398 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
376 }) 399 })
377 400
378 after(async function () { 401 after(async function () {
@@ -414,8 +437,8 @@ describe('Test videos redundancy', function () {
414 it('Should view 2 times the first video to have > min_views config', async function () { 437 it('Should view 2 times the first video to have > min_views config', async function () {
415 this.timeout(80000) 438 this.timeout(80000)
416 439
417 await servers[0].videos.view({ id: video1Server2UUID }) 440 await servers[0].videos.view({ id: video1Server2.uuid })
418 await servers[2].videos.view({ id: video1Server2UUID }) 441 await servers[2].videos.view({ id: video1Server2.uuid })
419 442
420 await wait(10000) 443 await wait(10000)
421 await waitJobs(servers) 444 await waitJobs(servers)
@@ -436,12 +459,13 @@ describe('Test videos redundancy', function () {
436 it('Should remove the video and the redundancy files', async function () { 459 it('Should remove the video and the redundancy files', async function () {
437 this.timeout(20000) 460 this.timeout(20000)
438 461
439 await servers[1].videos.remove({ id: video1Server2UUID }) 462 await saveVideoInServers(servers, video1Server2.uuid)
463 await servers[1].videos.remove({ id: video1Server2.uuid })
440 464
441 await waitJobs(servers) 465 await waitJobs(servers)
442 466
443 for (const server of servers) { 467 for (const server of servers) {
444 await checkVideoFilesWereRemoved(video1Server2UUID, server) 468 await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
445 } 469 }
446 }) 470 })
447 471
@@ -482,8 +506,8 @@ describe('Test videos redundancy', function () {
482 it('Should have 1 redundancy on the first video', async function () { 506 it('Should have 1 redundancy on the first video', async function () {
483 this.timeout(160000) 507 this.timeout(160000)
484 508
485 await servers[0].videos.view({ id: video1Server2UUID }) 509 await servers[0].videos.view({ id: video1Server2.uuid })
486 await servers[2].videos.view({ id: video1Server2UUID }) 510 await servers[2].videos.view({ id: video1Server2.uuid })
487 511
488 await wait(10000) 512 await wait(10000)
489 await waitJobs(servers) 513 await waitJobs(servers)
@@ -499,12 +523,13 @@ describe('Test videos redundancy', function () {
499 it('Should remove the video and the redundancy files', async function () { 523 it('Should remove the video and the redundancy files', async function () {
500 this.timeout(20000) 524 this.timeout(20000)
501 525
502 await servers[1].videos.remove({ id: video1Server2UUID }) 526 await saveVideoInServers(servers, video1Server2.uuid)
527 await servers[1].videos.remove({ id: video1Server2.uuid })
503 528
504 await waitJobs(servers) 529 await waitJobs(servers)
505 530
506 for (const server of servers) { 531 for (const server of servers) {
507 await checkVideoFilesWereRemoved(video1Server2UUID, server) 532 await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
508 } 533 }
509 }) 534 })
510 535
@@ -527,7 +552,7 @@ describe('Test videos redundancy', function () {
527 }) 552 })
528 553
529 it('Should create a redundancy on first video', async function () { 554 it('Should create a redundancy on first video', async function () {
530 await servers[0].redundancy.addVideo({ videoId: video1Server2Id }) 555 await servers[0].redundancy.addVideo({ videoId: video1Server2.id })
531 }) 556 })
532 557
533 it('Should have 2 webseeds on the first video', async function () { 558 it('Should have 2 webseeds on the first video', async function () {
@@ -562,7 +587,7 @@ describe('Test videos redundancy', function () {
562 await check1WebSeed() 587 await check1WebSeed()
563 await check0PlaylistRedundancies() 588 await check0PlaylistRedundancies()
564 589
565 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ]) 590 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
566 }) 591 })
567 592
568 after(async function () { 593 after(async function () {
@@ -575,7 +600,7 @@ describe('Test videos redundancy', function () {
575 600
576 async function checkContains (servers: PeerTubeServer[], str: string) { 601 async function checkContains (servers: PeerTubeServer[], str: string) {
577 for (const server of servers) { 602 for (const server of servers) {
578 const video = await server.videos.get({ id: video1Server2UUID }) 603 const video = await server.videos.get({ id: video1Server2.uuid })
579 604
580 for (const f of video.files) { 605 for (const f of video.files) {
581 expect(f.magnetUri).to.contain(str) 606 expect(f.magnetUri).to.contain(str)
@@ -585,7 +610,7 @@ describe('Test videos redundancy', function () {
585 610
586 async function checkNotContains (servers: PeerTubeServer[], str: string) { 611 async function checkNotContains (servers: PeerTubeServer[], str: string) {
587 for (const server of servers) { 612 for (const server of servers) {
588 const video = await server.videos.get({ id: video1Server2UUID }) 613 const video = await server.videos.get({ id: video1Server2.uuid })
589 614
590 for (const f of video.files) { 615 for (const f of video.files) {
591 expect(f.magnetUri).to.not.contain(str) 616 expect(f.magnetUri).to.not.contain(str)
@@ -646,8 +671,8 @@ describe('Test videos redundancy', function () {
646 await servers[0].servers.waitUntilLog('Duplicated ', 5) 671 await servers[0].servers.waitUntilLog('Duplicated ', 5)
647 await waitJobs(servers) 672 await waitJobs(servers)
648 673
649 await check2Webseeds(video1Server2UUID) 674 await check2Webseeds()
650 await check1PlaylistRedundancies(video1Server2UUID) 675 await check1PlaylistRedundancies()
651 await checkStatsWith1Redundancy(strategy) 676 await checkStatsWith1Redundancy(strategy)
652 677
653 const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } }) 678 const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
@@ -670,8 +695,8 @@ describe('Test videos redundancy', function () {
670 await wait(1000) 695 await wait(1000)
671 696
672 try { 697 try {
673 await check1WebSeed(video1Server2UUID) 698 await check1WebSeed()
674 await check0PlaylistRedundancies(video1Server2UUID) 699 await check0PlaylistRedundancies()
675 700
676 await check2Webseeds(video2Server2UUID) 701 await check2Webseeds(video2Server2UUID)
677 await check1PlaylistRedundancies(video2Server2UUID) 702 await check1PlaylistRedundancies(video2Server2UUID)
@@ -700,7 +725,7 @@ describe('Test videos redundancy', function () {
700 725
701 await waitJobs(servers) 726 await waitJobs(servers)
702 727
703 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ join('redundancy', 'hls') ]) 728 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
704 }) 729 })
705 730
706 after(async function () { 731 after(async function () {