1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { readdir } from 'fs-extra'
6 import * as magnetUtil from 'magnet-uri'
7 import { join } from 'path'
8 import { HttpStatusCode } from '@shared/core-utils'
11 checkVideoFilesWereRemoved,
14 flushAndRunMultipleServers,
20 setAccessTokensToServers,
23 } from '@shared/extra-utils'
24 import { VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models'
26 const expect = chai.expect
28 let servers: ServerInfo[] = []
29 let video1Server2UUID: string
30 let video1Server2Id: number
32 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
33 const parsed = magnetUtil.decode(file.magnetUri)
35 for (const ws of baseWebseeds) {
36 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
37 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
40 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
43 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
44 const strategies: any[] = []
46 if (strategy !== null) {
49 min_lifetime: '1 hour',
61 enabled: withWebtorrent
69 check_interval: '5 seconds',
75 servers = await flushAndRunMultipleServers(3, config)
77 // Get the access tokens
78 await setAccessTokensToServers(servers)
81 const { uuid, id } = await servers[1].videosCommand.upload({ attributes: { name: 'video 1 server 2' } })
82 video1Server2UUID = uuid
85 await servers[1].videosCommand.view({ id: video1Server2UUID })
88 await waitJobs(servers)
90 // Server 1 and server 2 follow each other
91 await doubleFollow(servers[0], servers[1])
92 // Server 1 and server 3 follow each other
93 await doubleFollow(servers[0], servers[2])
94 // Server 2 and server 3 follow each other
95 await doubleFollow(servers[1], servers[2])
97 await waitJobs(servers)
100 async function check1WebSeed (videoUUID?: string) {
101 if (!videoUUID) videoUUID = video1Server2UUID
104 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
107 for (const server of servers) {
108 // With token to avoid issues with video follow constraints
109 const video = await server.videosCommand.getWithToken({ id: videoUUID })
111 for (const f of video.files) {
112 checkMagnetWebseeds(f, webseeds, server)
117 async function check2Webseeds (videoUUID?: string) {
118 if (!videoUUID) videoUUID = video1Server2UUID
121 `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
122 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
125 for (const server of servers) {
126 const video = await server.videosCommand.get({ id: videoUUID })
128 for (const file of video.files) {
129 checkMagnetWebseeds(file, webseeds, server)
131 await makeGetRequest({
133 statusCodeExpected: HttpStatusCode.OK_200,
134 path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
137 await makeGetRequest({
139 statusCodeExpected: HttpStatusCode.OK_200,
140 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
146 const directories = [
147 'test' + servers[0].internalServerNumber + '/redundancy',
148 'test' + servers[1].internalServerNumber + '/videos'
151 for (const directory of directories) {
152 const files = await readdir(join(root(), directory))
153 expect(files).to.have.length.at.least(4)
155 for (const resolution of [ 240, 360, 480, 720 ]) {
156 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
161 async function check0PlaylistRedundancies (videoUUID?: string) {
162 if (!videoUUID) videoUUID = video1Server2UUID
164 for (const server of servers) {
165 // With token to avoid issues with video follow constraints
166 const video = await server.videosCommand.getWithToken({ id: videoUUID })
168 expect(video.streamingPlaylists).to.be.an('array')
169 expect(video.streamingPlaylists).to.have.lengthOf(1)
170 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
174 async function check1PlaylistRedundancies (videoUUID?: string) {
175 if (!videoUUID) videoUUID = video1Server2UUID
177 for (const server of servers) {
178 const video = await server.videosCommand.get({ id: videoUUID })
180 expect(video.streamingPlaylists).to.have.lengthOf(1)
181 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
183 const redundancy = video.streamingPlaylists[0].redundancies[0]
185 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
188 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
189 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
191 const video = await servers[0].videosCommand.get({ id: videoUUID })
192 const hlsPlaylist = video.streamingPlaylists[0]
194 for (const resolution of [ 240, 360, 480, 720 ]) {
195 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist })
198 const directories = [
199 'test' + servers[0].internalServerNumber + '/redundancy/hls',
200 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
203 for (const directory of directories) {
204 const files = await readdir(join(root(), directory, videoUUID))
205 expect(files).to.have.length.at.least(4)
207 for (const resolution of [ 240, 360, 480, 720 ]) {
208 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
210 expect(files.find(f => f === filename)).to.not.be.undefined
215 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
216 let totalSize: number = null
219 if (strategy !== 'manual') {
224 const data = await servers[0].statsCommand.get()
225 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
227 const stat = data.videosRedundancy[0]
228 expect(stat.strategy).to.equal(strategy)
229 expect(stat.totalSize).to.equal(totalSize)
234 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
235 const stat = await checkStatsGlobal(strategy)
237 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
238 expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
239 expect(stat.totalVideos).to.equal(1)
242 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
243 const stat = await checkStatsGlobal(strategy)
245 expect(stat.totalUsed).to.equal(0)
246 expect(stat.totalVideoFiles).to.equal(0)
247 expect(stat.totalVideos).to.equal(0)
250 async function findServerFollows () {
251 const body = await servers[0].followsCommand.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
252 const follows = body.data
253 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
254 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
256 return { server2, server3 }
259 async function enableRedundancyOnServer1 () {
260 await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
262 const { server2, server3 } = await findServerFollows()
264 expect(server3).to.not.be.undefined
265 expect(server3.following.hostRedundancyAllowed).to.be.false
267 expect(server2).to.not.be.undefined
268 expect(server2.following.hostRedundancyAllowed).to.be.true
271 async function disableRedundancyOnServer1 () {
272 await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
274 const { server2, server3 } = await findServerFollows()
276 expect(server3).to.not.be.undefined
277 expect(server3.following.hostRedundancyAllowed).to.be.false
279 expect(server2).to.not.be.undefined
280 expect(server2.following.hostRedundancyAllowed).to.be.false
283 describe('Test videos redundancy', function () {
285 describe('With most-views strategy', function () {
286 const strategy = 'most-views'
291 return flushAndRunServers(strategy)
294 it('Should have 1 webseed on the first video', async function () {
295 await check1WebSeed()
296 await check0PlaylistRedundancies()
297 await checkStatsWithoutRedundancy(strategy)
300 it('Should enable redundancy on server 1', function () {
301 return enableRedundancyOnServer1()
304 it('Should have 2 webseeds on the first video', async function () {
307 await waitJobs(servers)
308 await servers[0].serversCommand.waitUntilLog('Duplicated ', 5)
309 await waitJobs(servers)
311 await check2Webseeds()
312 await check1PlaylistRedundancies()
313 await checkStatsWith1Redundancy(strategy)
316 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
319 await disableRedundancyOnServer1()
321 await waitJobs(servers)
324 await check1WebSeed()
325 await check0PlaylistRedundancies()
327 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos', join('playlists', 'hls') ])
330 after(async function () {
331 return cleanupTests(servers)
335 describe('With trending strategy', function () {
336 const strategy = 'trending'
341 return flushAndRunServers(strategy)
344 it('Should have 1 webseed on the first video', async function () {
345 await check1WebSeed()
346 await check0PlaylistRedundancies()
347 await checkStatsWithoutRedundancy(strategy)
350 it('Should enable redundancy on server 1', function () {
351 return enableRedundancyOnServer1()
354 it('Should have 2 webseeds on the first video', async function () {
357 await waitJobs(servers)
358 await servers[0].serversCommand.waitUntilLog('Duplicated ', 5)
359 await waitJobs(servers)
361 await check2Webseeds()
362 await check1PlaylistRedundancies()
363 await checkStatsWith1Redundancy(strategy)
366 it('Should unfollow on server 1 and remove duplicated videos', async function () {
369 await servers[0].followsCommand.unfollow({ target: servers[1] })
371 await waitJobs(servers)
374 await check1WebSeed()
375 await check0PlaylistRedundancies()
377 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ])
380 after(async function () {
381 await cleanupTests(servers)
385 describe('With recently added strategy', function () {
386 const strategy = 'recently-added'
391 return flushAndRunServers(strategy, { min_views: 3 })
394 it('Should have 1 webseed on the first video', async function () {
395 await check1WebSeed()
396 await check0PlaylistRedundancies()
397 await checkStatsWithoutRedundancy(strategy)
400 it('Should enable redundancy on server 1', function () {
401 return enableRedundancyOnServer1()
404 it('Should still have 1 webseed on the first video', async function () {
407 await waitJobs(servers)
409 await waitJobs(servers)
411 await check1WebSeed()
412 await check0PlaylistRedundancies()
413 await checkStatsWithoutRedundancy(strategy)
416 it('Should view 2 times the first video to have > min_views config', async function () {
419 await servers[0].videosCommand.view({ id: video1Server2UUID })
420 await servers[2].videosCommand.view({ id: video1Server2UUID })
423 await waitJobs(servers)
426 it('Should have 2 webseeds on the first video', async function () {
429 await waitJobs(servers)
430 await servers[0].serversCommand.waitUntilLog('Duplicated ', 5)
431 await waitJobs(servers)
433 await check2Webseeds()
434 await check1PlaylistRedundancies()
435 await checkStatsWith1Redundancy(strategy)
438 it('Should remove the video and the redundancy files', async function () {
441 await servers[1].videosCommand.remove({ id: video1Server2UUID })
443 await waitJobs(servers)
445 for (const server of servers) {
446 await checkVideoFilesWereRemoved(video1Server2UUID, server)
450 after(async function () {
451 await cleanupTests(servers)
455 describe('With only HLS files', function () {
456 const strategy = 'recently-added'
458 before(async function () {
461 await flushAndRunServers(strategy, { min_views: 3 }, false)
464 it('Should have 0 playlist redundancy on the first video', async function () {
465 await check1WebSeed()
466 await check0PlaylistRedundancies()
469 it('Should enable redundancy on server 1', function () {
470 return enableRedundancyOnServer1()
473 it('Should still have 0 redundancy on the first video', async function () {
476 await waitJobs(servers)
478 await waitJobs(servers)
480 await check0PlaylistRedundancies()
481 await checkStatsWithoutRedundancy(strategy)
484 it('Should have 1 redundancy on the first video', async function () {
487 await servers[0].videosCommand.view({ id: video1Server2UUID })
488 await servers[2].videosCommand.view({ id: video1Server2UUID })
491 await waitJobs(servers)
493 await waitJobs(servers)
494 await servers[0].serversCommand.waitUntilLog('Duplicated ', 1)
495 await waitJobs(servers)
497 await check1PlaylistRedundancies()
498 await checkStatsWith1Redundancy(strategy, true)
501 it('Should remove the video and the redundancy files', async function () {
504 await servers[1].videosCommand.remove({ id: video1Server2UUID })
506 await waitJobs(servers)
508 for (const server of servers) {
509 await checkVideoFilesWereRemoved(video1Server2UUID, server)
513 after(async function () {
514 await cleanupTests(servers)
518 describe('With manual strategy', function () {
522 return flushAndRunServers(null)
525 it('Should have 1 webseed on the first video', async function () {
526 await check1WebSeed()
527 await check0PlaylistRedundancies()
528 await checkStatsWithoutRedundancy('manual')
531 it('Should create a redundancy on first video', async function () {
532 await servers[0].redundancyCommand.addVideo({ videoId: video1Server2Id })
535 it('Should have 2 webseeds on the first video', async function () {
538 await waitJobs(servers)
539 await servers[0].serversCommand.waitUntilLog('Duplicated ', 5)
540 await waitJobs(servers)
542 await check2Webseeds()
543 await check1PlaylistRedundancies()
544 await checkStatsWith1Redundancy('manual')
547 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
550 const body = await servers[0].redundancyCommand.listVideos({ target: 'remote-videos' })
552 const videos = body.data
553 expect(videos).to.have.lengthOf(1)
555 const video = videos[0]
557 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
558 await servers[0].redundancyCommand.removeVideo({ redundancyId: r.id })
561 await waitJobs(servers)
564 await check1WebSeed()
565 await check0PlaylistRedundancies()
567 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ 'videos' ])
570 after(async function () {
571 await cleanupTests(servers)
575 describe('Test expiration', function () {
576 const strategy = 'recently-added'
578 async function checkContains (servers: ServerInfo[], str: string) {
579 for (const server of servers) {
580 const video = await server.videosCommand.get({ id: video1Server2UUID })
582 for (const f of video.files) {
583 expect(f.magnetUri).to.contain(str)
588 async function checkNotContains (servers: ServerInfo[], str: string) {
589 for (const server of servers) {
590 const video = await server.videosCommand.get({ id: video1Server2UUID })
592 for (const f of video.files) {
593 expect(f.magnetUri).to.not.contain(str)
598 before(async function () {
601 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
603 await enableRedundancyOnServer1()
606 it('Should still have 2 webseeds after 10 seconds', async function () {
612 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
614 // Maybe a server deleted a redundancy in the scheduler
617 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
621 it('Should stop server 1 and expire video redundancy', async function () {
624 await killallServers([ servers[0] ])
628 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
631 after(async function () {
632 await cleanupTests(servers)
636 describe('Test file replacement', function () {
637 let video2Server2UUID: string
638 const strategy = 'recently-added'
640 before(async function () {
643 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
645 await enableRedundancyOnServer1()
647 await waitJobs(servers)
648 await servers[0].serversCommand.waitUntilLog('Duplicated ', 5)
649 await waitJobs(servers)
651 await check2Webseeds(video1Server2UUID)
652 await check1PlaylistRedundancies(video1Server2UUID)
653 await checkStatsWith1Redundancy(strategy)
655 const { uuid } = await servers[1].videosCommand.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
656 video2Server2UUID = uuid
658 // Wait transcoding before federation
659 await waitJobs(servers)
661 await servers[1].videosCommand.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
664 it('Should cache video 2 webseeds on the first video', async function () {
667 await waitJobs(servers)
671 while (checked === false) {
675 await check1WebSeed(video1Server2UUID)
676 await check0PlaylistRedundancies(video1Server2UUID)
678 await check2Webseeds(video2Server2UUID)
679 await check1PlaylistRedundancies(video2Server2UUID)
688 it('Should disable strategy and remove redundancies', async function () {
691 await waitJobs(servers)
693 await killallServers([ servers[0] ])
694 await reRunServer(servers[0], {
697 check_interval: '1 second',
703 await waitJobs(servers)
705 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0], [ join('redundancy', 'hls') ])
708 after(async function () {
709 await cleanupTests(servers)