1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { readdir } from 'fs-extra'
6 import * as magnetUtil from 'magnet-uri'
7 import { join } from 'path'
8 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
11 checkVideoFilesWereRemoved,
14 flushAndRunMultipleServers,
15 getFollowingListPaginationAndSort,
25 setAccessTokensToServers,
31 } from '../../../../shared/extra-utils'
32 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
35 listVideoRedundancies,
36 removeVideoRedundancy,
38 } from '../../../../shared/extra-utils/server/redundancy'
39 import { getStats } from '../../../../shared/extra-utils/server/stats'
40 import { ActorFollow } from '../../../../shared/models/actors'
41 import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
42 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
43 import { VideoDetails } from '../../../../shared/models/videos'
45 const expect = chai.expect
47 let servers: ServerInfo[] = []
48 let video1Server2UUID: string
49 let video1Server2Id: number
51 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
52 const parsed = magnetUtil.decode(file.magnetUri)
54 for (const ws of baseWebseeds) {
55 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
56 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
59 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
62 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
63 const strategies: any[] = []
65 if (strategy !== null) {
68 min_lifetime: '1 hour',
78 enabled: withWebtorrent
86 check_interval: '5 seconds',
92 servers = await flushAndRunMultipleServers(3, config)
94 // Get the access tokens
95 await setAccessTokensToServers(servers)
98 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
99 video1Server2UUID = res.body.video.uuid
100 video1Server2Id = res.body.video.id
102 await viewVideo(servers[1].url, video1Server2UUID)
105 await waitJobs(servers)
107 // Server 1 and server 2 follow each other
108 await doubleFollow(servers[0], servers[1])
109 // Server 1 and server 3 follow each other
110 await doubleFollow(servers[0], servers[2])
111 // Server 2 and server 3 follow each other
112 await doubleFollow(servers[1], servers[2])
114 await waitJobs(servers)
117 async function check1WebSeed (videoUUID?: string) {
118 if (!videoUUID) videoUUID = video1Server2UUID
121 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
124 for (const server of servers) {
125 // With token to avoid issues with video follow constraints
126 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
128 const video: VideoDetails = res.body
129 for (const f of video.files) {
130 checkMagnetWebseeds(f, webseeds, server)
135 async function check2Webseeds (videoUUID?: string) {
136 if (!videoUUID) videoUUID = video1Server2UUID
139 `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
140 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
143 for (const server of servers) {
144 const res = await getVideo(server.url, videoUUID)
146 const video: VideoDetails = res.body
148 for (const file of video.files) {
149 checkMagnetWebseeds(file, webseeds, server)
151 await makeGetRequest({
153 statusCodeExpected: HttpStatusCode.OK_200,
154 path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
157 await makeGetRequest({
159 statusCodeExpected: HttpStatusCode.OK_200,
160 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
166 const directories = [
167 'test' + servers[0].internalServerNumber + '/redundancy',
168 'test' + servers[1].internalServerNumber + '/videos'
171 for (const directory of directories) {
172 const files = await readdir(join(root(), directory))
173 expect(files).to.have.length.at.least(4)
175 for (const resolution of [ 240, 360, 480, 720 ]) {
176 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
181 async function check0PlaylistRedundancies (videoUUID?: string) {
182 if (!videoUUID) videoUUID = video1Server2UUID
184 for (const server of servers) {
185 // With token to avoid issues with video follow constraints
186 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
187 const video: VideoDetails = res.body
189 expect(video.streamingPlaylists).to.be.an('array')
190 expect(video.streamingPlaylists).to.have.lengthOf(1)
191 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
195 async function check1PlaylistRedundancies (videoUUID?: string) {
196 if (!videoUUID) videoUUID = video1Server2UUID
198 for (const server of servers) {
199 const res = await getVideo(server.url, videoUUID)
200 const video: VideoDetails = res.body
202 expect(video.streamingPlaylists).to.have.lengthOf(1)
203 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
205 const redundancy = video.streamingPlaylists[0].redundancies[0]
207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
213 const res = await getVideo(servers[0].url, videoUUID)
214 const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
216 for (const resolution of [ 240, 360, 480, 720 ]) {
217 await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
220 const directories = [
221 'test' + servers[0].internalServerNumber + '/redundancy/hls',
222 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
225 for (const directory of directories) {
226 const files = await readdir(join(root(), directory, videoUUID))
227 expect(files).to.have.length.at.least(4)
229 for (const resolution of [ 240, 360, 480, 720 ]) {
230 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
232 expect(files.find(f => f === filename)).to.not.be.undefined
237 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
238 let totalSize: number = null
241 if (strategy !== 'manual') {
246 const res = await getStats(servers[0].url)
247 const data: ServerStats = res.body
249 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
251 const stat = data.videosRedundancy[0]
252 expect(stat.strategy).to.equal(strategy)
253 expect(stat.totalSize).to.equal(totalSize)
258 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
259 const stat = await checkStatsGlobal(strategy)
261 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
262 expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
263 expect(stat.totalVideos).to.equal(1)
266 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
267 const stat = await checkStatsGlobal(strategy)
269 expect(stat.totalUsed).to.equal(0)
270 expect(stat.totalVideoFiles).to.equal(0)
271 expect(stat.totalVideos).to.equal(0)
274 async function enableRedundancyOnServer1 () {
275 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
277 const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
278 const follows: ActorFollow[] = res.body.data
279 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
280 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
282 expect(server3).to.not.be.undefined
283 expect(server3.following.hostRedundancyAllowed).to.be.false
285 expect(server2).to.not.be.undefined
286 expect(server2.following.hostRedundancyAllowed).to.be.true
289 async function disableRedundancyOnServer1 () {
290 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
292 const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
293 const follows: ActorFollow[] = res.body.data
294 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
295 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
297 expect(server3).to.not.be.undefined
298 expect(server3.following.hostRedundancyAllowed).to.be.false
300 expect(server2).to.not.be.undefined
301 expect(server2.following.hostRedundancyAllowed).to.be.false
304 describe('Test videos redundancy', function () {
306 describe('With most-views strategy', function () {
307 const strategy = 'most-views'
312 return flushAndRunServers(strategy)
315 it('Should have 1 webseed on the first video', async function () {
316 await check1WebSeed()
317 await check0PlaylistRedundancies()
318 await checkStatsWithoutRedundancy(strategy)
321 it('Should enable redundancy on server 1', function () {
322 return enableRedundancyOnServer1()
325 it('Should have 2 webseeds on the first video', async function () {
328 await waitJobs(servers)
329 await waitUntilLog(servers[0], 'Duplicated ', 5)
330 await waitJobs(servers)
332 await check2Webseeds()
333 await check1PlaylistRedundancies()
334 await checkStatsWith1Redundancy(strategy)
337 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
340 await disableRedundancyOnServer1()
342 await waitJobs(servers)
345 await check1WebSeed()
346 await check0PlaylistRedundancies()
348 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
351 after(async function () {
352 return cleanupTests(servers)
356 describe('With trending strategy', function () {
357 const strategy = 'trending'
362 return flushAndRunServers(strategy)
365 it('Should have 1 webseed on the first video', async function () {
366 await check1WebSeed()
367 await check0PlaylistRedundancies()
368 await checkStatsWithoutRedundancy(strategy)
371 it('Should enable redundancy on server 1', function () {
372 return enableRedundancyOnServer1()
375 it('Should have 2 webseeds on the first video', async function () {
378 await waitJobs(servers)
379 await waitUntilLog(servers[0], 'Duplicated ', 5)
380 await waitJobs(servers)
382 await check2Webseeds()
383 await check1PlaylistRedundancies()
384 await checkStatsWith1Redundancy(strategy)
387 it('Should unfollow on server 1 and remove duplicated videos', async function () {
390 await unfollow(servers[0].url, servers[0].accessToken, servers[1])
392 await waitJobs(servers)
395 await check1WebSeed()
396 await check0PlaylistRedundancies()
398 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
401 after(async function () {
402 await cleanupTests(servers)
406 describe('With recently added strategy', function () {
407 const strategy = 'recently-added'
412 return flushAndRunServers(strategy, { min_views: 3 })
415 it('Should have 1 webseed on the first video', async function () {
416 await check1WebSeed()
417 await check0PlaylistRedundancies()
418 await checkStatsWithoutRedundancy(strategy)
421 it('Should enable redundancy on server 1', function () {
422 return enableRedundancyOnServer1()
425 it('Should still have 1 webseed on the first video', async function () {
428 await waitJobs(servers)
430 await waitJobs(servers)
432 await check1WebSeed()
433 await check0PlaylistRedundancies()
434 await checkStatsWithoutRedundancy(strategy)
437 it('Should view 2 times the first video to have > min_views config', async function () {
440 await viewVideo(servers[0].url, video1Server2UUID)
441 await viewVideo(servers[2].url, video1Server2UUID)
444 await waitJobs(servers)
447 it('Should have 2 webseeds on the first video', async function () {
450 await waitJobs(servers)
451 await waitUntilLog(servers[0], 'Duplicated ', 5)
452 await waitJobs(servers)
454 await check2Webseeds()
455 await check1PlaylistRedundancies()
456 await checkStatsWith1Redundancy(strategy)
459 it('Should remove the video and the redundancy files', async function () {
462 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
464 await waitJobs(servers)
466 for (const server of servers) {
467 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
471 after(async function () {
472 await cleanupTests(servers)
476 describe('With only HLS files', function () {
477 const strategy = 'recently-added'
479 before(async function () {
482 await flushAndRunServers(strategy, { min_views: 3 }, false)
485 it('Should have 0 playlist redundancy on the first video', async function () {
486 await check1WebSeed()
487 await check0PlaylistRedundancies()
490 it('Should enable redundancy on server 1', function () {
491 return enableRedundancyOnServer1()
494 it('Should still have 0 redundancy on the first video', async function () {
497 await waitJobs(servers)
499 await waitJobs(servers)
501 await check0PlaylistRedundancies()
502 await checkStatsWithoutRedundancy(strategy)
505 it('Should have 1 redundancy on the first video', async function () {
508 await viewVideo(servers[0].url, video1Server2UUID)
509 await viewVideo(servers[2].url, video1Server2UUID)
512 await waitJobs(servers)
514 await waitJobs(servers)
515 await waitUntilLog(servers[0], 'Duplicated ', 1)
516 await waitJobs(servers)
518 await check1PlaylistRedundancies()
519 await checkStatsWith1Redundancy(strategy, true)
522 it('Should remove the video and the redundancy files', async function () {
525 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
527 await waitJobs(servers)
529 for (const server of servers) {
530 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
534 after(async function () {
535 await cleanupTests(servers)
539 describe('With manual strategy', function () {
543 return flushAndRunServers(null)
546 it('Should have 1 webseed on the first video', async function () {
547 await check1WebSeed()
548 await check0PlaylistRedundancies()
549 await checkStatsWithoutRedundancy('manual')
552 it('Should create a redundancy on first video', async function () {
553 await addVideoRedundancy({
555 accessToken: servers[0].accessToken,
556 videoId: video1Server2Id
560 it('Should have 2 webseeds on the first video', async function () {
563 await waitJobs(servers)
564 await waitUntilLog(servers[0], 'Duplicated ', 5)
565 await waitJobs(servers)
567 await check2Webseeds()
568 await check1PlaylistRedundancies()
569 await checkStatsWith1Redundancy('manual')
572 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
575 const res = await listVideoRedundancies({
577 accessToken: servers[0].accessToken,
578 target: 'remote-videos'
581 const videos = res.body.data as VideoRedundancy[]
582 expect(videos).to.have.lengthOf(1)
584 const video = videos[0]
585 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
586 await removeVideoRedundancy({
588 accessToken: servers[0].accessToken,
593 await waitJobs(servers)
596 await check1WebSeed()
597 await check0PlaylistRedundancies()
599 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
602 after(async function () {
603 await cleanupTests(servers)
607 describe('Test expiration', function () {
608 const strategy = 'recently-added'
610 async function checkContains (servers: ServerInfo[], str: string) {
611 for (const server of servers) {
612 const res = await getVideo(server.url, video1Server2UUID)
613 const video: VideoDetails = res.body
615 for (const f of video.files) {
616 expect(f.magnetUri).to.contain(str)
621 async function checkNotContains (servers: ServerInfo[], str: string) {
622 for (const server of servers) {
623 const res = await getVideo(server.url, video1Server2UUID)
624 const video: VideoDetails = res.body
626 for (const f of video.files) {
627 expect(f.magnetUri).to.not.contain(str)
632 before(async function () {
635 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
637 await enableRedundancyOnServer1()
640 it('Should still have 2 webseeds after 10 seconds', async function () {
646 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
648 // Maybe a server deleted a redundancy in the scheduler
651 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
655 it('Should stop server 1 and expire video redundancy', async function () {
658 killallServers([ servers[0] ])
662 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
665 after(async function () {
666 await cleanupTests(servers)
670 describe('Test file replacement', function () {
671 let video2Server2UUID: string
672 const strategy = 'recently-added'
674 before(async function () {
677 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
679 await enableRedundancyOnServer1()
681 await waitJobs(servers)
682 await waitUntilLog(servers[0], 'Duplicated ', 5)
683 await waitJobs(servers)
685 await check2Webseeds()
686 await check1PlaylistRedundancies()
687 await checkStatsWith1Redundancy(strategy)
689 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' })
690 video2Server2UUID = res.body.video.uuid
693 it('Should cache video 2 webseeds on the first video', async function () {
696 await waitJobs(servers)
700 while (checked === false) {
704 await check1WebSeed(video1Server2UUID)
705 await check0PlaylistRedundancies(video1Server2UUID)
706 await check2Webseeds(video2Server2UUID)
707 await check1PlaylistRedundancies(video2Server2UUID)
716 it('Should disable strategy and remove redundancies', async function () {
719 await waitJobs(servers)
721 killallServers([ servers[0] ])
722 await reRunServer(servers[0], {
725 check_interval: '1 second',
731 await waitJobs(servers)
733 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
736 after(async function () {
737 await cleanupTests(servers)