1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { readdir } from 'fs-extra'
6 import * as magnetUtil from 'magnet-uri'
7 import { join } from 'path'
8 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
11 checkVideoFilesWereRemoved,
14 flushAndRunMultipleServers,
15 getFollowingListPaginationAndSort,
25 setAccessTokensToServers,
32 } from '../../../../shared/extra-utils'
33 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
36 listVideoRedundancies,
37 removeVideoRedundancy,
39 } from '../../../../shared/extra-utils/server/redundancy'
40 import { getStats } from '../../../../shared/extra-utils/server/stats'
41 import { ActorFollow } from '../../../../shared/models/actors'
42 import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
43 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
44 import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos'
46 const expect = chai.expect
48 let servers: ServerInfo[] = []
49 let video1Server2UUID: string
50 let video1Server2Id: number
52 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
53 const parsed = magnetUtil.decode(file.magnetUri)
55 for (const ws of baseWebseeds) {
56 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
57 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
60 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
63 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
64 const strategies: any[] = []
66 if (strategy !== null) {
69 min_lifetime: '1 hour',
79 enabled: withWebtorrent
87 check_interval: '5 seconds',
93 servers = await flushAndRunMultipleServers(3, config)
95 // Get the access tokens
96 await setAccessTokensToServers(servers)
99 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
100 video1Server2UUID = res.body.video.uuid
101 video1Server2Id = res.body.video.id
103 await viewVideo(servers[1].url, video1Server2UUID)
106 await waitJobs(servers)
108 // Server 1 and server 2 follow each other
109 await doubleFollow(servers[0], servers[1])
110 // Server 1 and server 3 follow each other
111 await doubleFollow(servers[0], servers[2])
112 // Server 2 and server 3 follow each other
113 await doubleFollow(servers[1], servers[2])
115 await waitJobs(servers)
118 async function check1WebSeed (videoUUID?: string) {
119 if (!videoUUID) videoUUID = video1Server2UUID
122 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
125 for (const server of servers) {
126 // With token to avoid issues with video follow constraints
127 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
129 const video: VideoDetails = res.body
130 for (const f of video.files) {
131 checkMagnetWebseeds(f, webseeds, server)
136 async function check2Webseeds (videoUUID?: string) {
137 if (!videoUUID) videoUUID = video1Server2UUID
140 `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
141 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
144 for (const server of servers) {
145 const res = await getVideo(server.url, videoUUID)
147 const video: VideoDetails = res.body
149 for (const file of video.files) {
150 checkMagnetWebseeds(file, webseeds, server)
152 await makeGetRequest({
154 statusCodeExpected: HttpStatusCode.OK_200,
155 path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
158 await makeGetRequest({
160 statusCodeExpected: HttpStatusCode.OK_200,
161 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
167 const directories = [
168 'test' + servers[0].internalServerNumber + '/redundancy',
169 'test' + servers[1].internalServerNumber + '/videos'
172 for (const directory of directories) {
173 const files = await readdir(join(root(), directory))
174 expect(files).to.have.length.at.least(4)
176 for (const resolution of [ 240, 360, 480, 720 ]) {
177 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
182 async function check0PlaylistRedundancies (videoUUID?: string) {
183 if (!videoUUID) videoUUID = video1Server2UUID
185 for (const server of servers) {
186 // With token to avoid issues with video follow constraints
187 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
188 const video: VideoDetails = res.body
190 expect(video.streamingPlaylists).to.be.an('array')
191 expect(video.streamingPlaylists).to.have.lengthOf(1)
192 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
196 async function check1PlaylistRedundancies (videoUUID?: string) {
197 if (!videoUUID) videoUUID = video1Server2UUID
199 for (const server of servers) {
200 const res = await getVideo(server.url, videoUUID)
201 const video: VideoDetails = res.body
203 expect(video.streamingPlaylists).to.have.lengthOf(1)
204 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
206 const redundancy = video.streamingPlaylists[0].redundancies[0]
208 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
211 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
212 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
214 const res = await getVideo(servers[0].url, videoUUID)
215 const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
217 for (const resolution of [ 240, 360, 480, 720 ]) {
218 await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
221 const directories = [
222 'test' + servers[0].internalServerNumber + '/redundancy/hls',
223 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
226 for (const directory of directories) {
227 const files = await readdir(join(root(), directory, videoUUID))
228 expect(files).to.have.length.at.least(4)
230 for (const resolution of [ 240, 360, 480, 720 ]) {
231 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
233 expect(files.find(f => f === filename)).to.not.be.undefined
238 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
239 let totalSize: number = null
242 if (strategy !== 'manual') {
247 const res = await getStats(servers[0].url)
248 const data: ServerStats = res.body
250 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
252 const stat = data.videosRedundancy[0]
253 expect(stat.strategy).to.equal(strategy)
254 expect(stat.totalSize).to.equal(totalSize)
259 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
260 const stat = await checkStatsGlobal(strategy)
262 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
263 expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
264 expect(stat.totalVideos).to.equal(1)
267 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
268 const stat = await checkStatsGlobal(strategy)
270 expect(stat.totalUsed).to.equal(0)
271 expect(stat.totalVideoFiles).to.equal(0)
272 expect(stat.totalVideos).to.equal(0)
275 async function enableRedundancyOnServer1 () {
276 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
278 const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
279 const follows: ActorFollow[] = res.body.data
280 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
281 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
283 expect(server3).to.not.be.undefined
284 expect(server3.following.hostRedundancyAllowed).to.be.false
286 expect(server2).to.not.be.undefined
287 expect(server2.following.hostRedundancyAllowed).to.be.true
290 async function disableRedundancyOnServer1 () {
291 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
293 const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' })
294 const follows: ActorFollow[] = res.body.data
295 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
296 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
298 expect(server3).to.not.be.undefined
299 expect(server3.following.hostRedundancyAllowed).to.be.false
301 expect(server2).to.not.be.undefined
302 expect(server2.following.hostRedundancyAllowed).to.be.false
305 describe('Test videos redundancy', function () {
307 describe('With most-views strategy', function () {
308 const strategy = 'most-views'
313 return flushAndRunServers(strategy)
316 it('Should have 1 webseed on the first video', async function () {
317 await check1WebSeed()
318 await check0PlaylistRedundancies()
319 await checkStatsWithoutRedundancy(strategy)
322 it('Should enable redundancy on server 1', function () {
323 return enableRedundancyOnServer1()
326 it('Should have 2 webseeds on the first video', async function () {
329 await waitJobs(servers)
330 await waitUntilLog(servers[0], 'Duplicated ', 5)
331 await waitJobs(servers)
333 await check2Webseeds()
334 await check1PlaylistRedundancies()
335 await checkStatsWith1Redundancy(strategy)
338 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
341 await disableRedundancyOnServer1()
343 await waitJobs(servers)
346 await check1WebSeed()
347 await check0PlaylistRedundancies()
349 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
352 after(async function () {
353 return cleanupTests(servers)
357 describe('With trending strategy', function () {
358 const strategy = 'trending'
363 return flushAndRunServers(strategy)
366 it('Should have 1 webseed on the first video', async function () {
367 await check1WebSeed()
368 await check0PlaylistRedundancies()
369 await checkStatsWithoutRedundancy(strategy)
372 it('Should enable redundancy on server 1', function () {
373 return enableRedundancyOnServer1()
376 it('Should have 2 webseeds on the first video', async function () {
379 await waitJobs(servers)
380 await waitUntilLog(servers[0], 'Duplicated ', 5)
381 await waitJobs(servers)
383 await check2Webseeds()
384 await check1PlaylistRedundancies()
385 await checkStatsWith1Redundancy(strategy)
388 it('Should unfollow on server 1 and remove duplicated videos', async function () {
391 await unfollow(servers[0].url, servers[0].accessToken, servers[1])
393 await waitJobs(servers)
396 await check1WebSeed()
397 await check0PlaylistRedundancies()
399 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
402 after(async function () {
403 await cleanupTests(servers)
407 describe('With recently added strategy', function () {
408 const strategy = 'recently-added'
413 return flushAndRunServers(strategy, { min_views: 3 })
416 it('Should have 1 webseed on the first video', async function () {
417 await check1WebSeed()
418 await check0PlaylistRedundancies()
419 await checkStatsWithoutRedundancy(strategy)
422 it('Should enable redundancy on server 1', function () {
423 return enableRedundancyOnServer1()
426 it('Should still have 1 webseed on the first video', async function () {
429 await waitJobs(servers)
431 await waitJobs(servers)
433 await check1WebSeed()
434 await check0PlaylistRedundancies()
435 await checkStatsWithoutRedundancy(strategy)
438 it('Should view 2 times the first video to have > min_views config', async function () {
441 await viewVideo(servers[0].url, video1Server2UUID)
442 await viewVideo(servers[2].url, video1Server2UUID)
445 await waitJobs(servers)
448 it('Should have 2 webseeds on the first video', async function () {
451 await waitJobs(servers)
452 await waitUntilLog(servers[0], 'Duplicated ', 5)
453 await waitJobs(servers)
455 await check2Webseeds()
456 await check1PlaylistRedundancies()
457 await checkStatsWith1Redundancy(strategy)
460 it('Should remove the video and the redundancy files', async function () {
463 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
465 await waitJobs(servers)
467 for (const server of servers) {
468 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
472 after(async function () {
473 await cleanupTests(servers)
477 describe('With only HLS files', function () {
478 const strategy = 'recently-added'
480 before(async function () {
483 await flushAndRunServers(strategy, { min_views: 3 }, false)
486 it('Should have 0 playlist redundancy on the first video', async function () {
487 await check1WebSeed()
488 await check0PlaylistRedundancies()
491 it('Should enable redundancy on server 1', function () {
492 return enableRedundancyOnServer1()
495 it('Should still have 0 redundancy on the first video', async function () {
498 await waitJobs(servers)
500 await waitJobs(servers)
502 await check0PlaylistRedundancies()
503 await checkStatsWithoutRedundancy(strategy)
506 it('Should have 1 redundancy on the first video', async function () {
509 await viewVideo(servers[0].url, video1Server2UUID)
510 await viewVideo(servers[2].url, video1Server2UUID)
513 await waitJobs(servers)
515 await waitJobs(servers)
516 await waitUntilLog(servers[0], 'Duplicated ', 1)
517 await waitJobs(servers)
519 await check1PlaylistRedundancies()
520 await checkStatsWith1Redundancy(strategy, true)
523 it('Should remove the video and the redundancy files', async function () {
526 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
528 await waitJobs(servers)
530 for (const server of servers) {
531 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
535 after(async function () {
536 await cleanupTests(servers)
540 describe('With manual strategy', function () {
544 return flushAndRunServers(null)
547 it('Should have 1 webseed on the first video', async function () {
548 await check1WebSeed()
549 await check0PlaylistRedundancies()
550 await checkStatsWithoutRedundancy('manual')
553 it('Should create a redundancy on first video', async function () {
554 await addVideoRedundancy({
556 accessToken: servers[0].accessToken,
557 videoId: video1Server2Id
561 it('Should have 2 webseeds on the first video', async function () {
564 await waitJobs(servers)
565 await waitUntilLog(servers[0], 'Duplicated ', 5)
566 await waitJobs(servers)
568 await check2Webseeds()
569 await check1PlaylistRedundancies()
570 await checkStatsWith1Redundancy('manual')
573 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
576 const res = await listVideoRedundancies({
578 accessToken: servers[0].accessToken,
579 target: 'remote-videos'
582 const videos = res.body.data as VideoRedundancy[]
583 expect(videos).to.have.lengthOf(1)
585 const video = videos[0]
586 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
587 await removeVideoRedundancy({
589 accessToken: servers[0].accessToken,
594 await waitJobs(servers)
597 await check1WebSeed()
598 await check0PlaylistRedundancies()
600 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
603 after(async function () {
604 await cleanupTests(servers)
608 describe('Test expiration', function () {
609 const strategy = 'recently-added'
611 async function checkContains (servers: ServerInfo[], str: string) {
612 for (const server of servers) {
613 const res = await getVideo(server.url, video1Server2UUID)
614 const video: VideoDetails = res.body
616 for (const f of video.files) {
617 expect(f.magnetUri).to.contain(str)
622 async function checkNotContains (servers: ServerInfo[], str: string) {
623 for (const server of servers) {
624 const res = await getVideo(server.url, video1Server2UUID)
625 const video: VideoDetails = res.body
627 for (const f of video.files) {
628 expect(f.magnetUri).to.not.contain(str)
633 before(async function () {
636 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
638 await enableRedundancyOnServer1()
641 it('Should still have 2 webseeds after 10 seconds', async function () {
647 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
649 // Maybe a server deleted a redundancy in the scheduler
652 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
656 it('Should stop server 1 and expire video redundancy', async function () {
659 killallServers([ servers[0] ])
663 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
666 after(async function () {
667 await cleanupTests(servers)
671 describe('Test file replacement', function () {
672 let video2Server2UUID: string
673 const strategy = 'recently-added'
675 before(async function () {
678 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
680 await enableRedundancyOnServer1()
682 await waitJobs(servers)
683 await waitUntilLog(servers[0], 'Duplicated ', 5)
684 await waitJobs(servers)
686 await check2Webseeds(video1Server2UUID)
687 await check1PlaylistRedundancies(video1Server2UUID)
688 await checkStatsWith1Redundancy(strategy)
690 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE })
691 video2Server2UUID = res.body.video.uuid
693 // Wait transcoding before federation
694 await waitJobs(servers)
696 await updateVideo(servers[1].url, servers[1].accessToken, video2Server2UUID, { privacy: VideoPrivacy.PUBLIC })
699 it('Should cache video 2 webseeds on the first video', async function () {
702 await waitJobs(servers)
706 while (checked === false) {
710 await check1WebSeed(video1Server2UUID)
711 await check0PlaylistRedundancies(video1Server2UUID)
713 await check2Webseeds(video2Server2UUID)
714 await check1PlaylistRedundancies(video2Server2UUID)
723 it('Should disable strategy and remove redundancies', async function () {
726 await waitJobs(servers)
728 killallServers([ servers[0] ])
729 await reRunServer(servers[0], {
732 check_interval: '1 second',
738 await waitJobs(servers)
740 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
743 after(async function () {
744 await cleanupTests(servers)