1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { readdir } from 'fs-extra'
6 import * as magnetUtil from 'magnet-uri'
7 import { join } from 'path'
8 import { HttpStatusCode } from '@shared/core-utils'
11 checkVideoFilesWereRemoved,
14 flushAndRunMultipleServers,
24 setAccessTokensToServers,
31 } from '@shared/extra-utils'
32 import { VideoDetails, VideoPrivacy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '@shared/models'
34 const expect = chai.expect
36 let servers: ServerInfo[] = []
37 let video1Server2UUID: string
38 let video1Server2Id: number
40 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
41 const parsed = magnetUtil.decode(file.magnetUri)
43 for (const ws of baseWebseeds) {
44 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
45 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
48 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
51 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
52 const strategies: any[] = []
54 if (strategy !== null) {
57 min_lifetime: '1 hour',
67 enabled: withWebtorrent
75 check_interval: '5 seconds',
81 servers = await flushAndRunMultipleServers(3, config)
83 // Get the access tokens
84 await setAccessTokensToServers(servers)
87 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
88 video1Server2UUID = res.body.video.uuid
89 video1Server2Id = res.body.video.id
91 await viewVideo(servers[1].url, video1Server2UUID)
94 await waitJobs(servers)
96 // Server 1 and server 2 follow each other
97 await doubleFollow(servers[0], servers[1])
98 // Server 1 and server 3 follow each other
99 await doubleFollow(servers[0], servers[2])
100 // Server 2 and server 3 follow each other
101 await doubleFollow(servers[1], servers[2])
103 await waitJobs(servers)
106 async function check1WebSeed (videoUUID?: string) {
107 if (!videoUUID) videoUUID = video1Server2UUID
110 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
113 for (const server of servers) {
114 // With token to avoid issues with video follow constraints
115 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
117 const video: VideoDetails = res.body
118 for (const f of video.files) {
119 checkMagnetWebseeds(f, webseeds, server)
124 async function check2Webseeds (videoUUID?: string) {
125 if (!videoUUID) videoUUID = video1Server2UUID
128 `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
129 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
132 for (const server of servers) {
133 const res = await getVideo(server.url, videoUUID)
135 const video: VideoDetails = res.body
137 for (const file of video.files) {
138 checkMagnetWebseeds(file, webseeds, server)
140 await makeGetRequest({
142 statusCodeExpected: HttpStatusCode.OK_200,
143 path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
146 await makeGetRequest({
148 statusCodeExpected: HttpStatusCode.OK_200,
149 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
155 const directories = [
156 'test' + servers[0].internalServerNumber + '/redundancy',
157 'test' + servers[1].internalServerNumber + '/videos'
160 for (const directory of directories) {
161 const files = await readdir(join(root(), directory))
162 expect(files).to.have.length.at.least(4)
164 for (const resolution of [ 240, 360, 480, 720 ]) {
165 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
170 async function check0PlaylistRedundancies (videoUUID?: string) {
171 if (!videoUUID) videoUUID = video1Server2UUID
173 for (const server of servers) {
174 // With token to avoid issues with video follow constraints
175 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
176 const video: VideoDetails = res.body
178 expect(video.streamingPlaylists).to.be.an('array')
179 expect(video.streamingPlaylists).to.have.lengthOf(1)
180 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
184 async function check1PlaylistRedundancies (videoUUID?: string) {
185 if (!videoUUID) videoUUID = video1Server2UUID
187 for (const server of servers) {
188 const res = await getVideo(server.url, videoUUID)
189 const video: VideoDetails = res.body
191 expect(video.streamingPlaylists).to.have.lengthOf(1)
192 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
194 const redundancy = video.streamingPlaylists[0].redundancies[0]
196 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
199 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
200 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
202 const res = await getVideo(servers[0].url, videoUUID)
203 const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
205 for (const resolution of [ 240, 360, 480, 720 ]) {
206 await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
209 const directories = [
210 'test' + servers[0].internalServerNumber + '/redundancy/hls',
211 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
214 for (const directory of directories) {
215 const files = await readdir(join(root(), directory, videoUUID))
216 expect(files).to.have.length.at.least(4)
218 for (const resolution of [ 240, 360, 480, 720 ]) {
219 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
221 expect(files.find(f => f === filename)).to.not.be.undefined
226 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
227 let totalSize: number = null
230 if (strategy !== 'manual') {
235 const data = await servers[0].statsCommand.get()
236 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
238 const stat = data.videosRedundancy[0]
239 expect(stat.strategy).to.equal(strategy)
240 expect(stat.totalSize).to.equal(totalSize)
245 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
246 const stat = await checkStatsGlobal(strategy)
248 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
249 expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
250 expect(stat.totalVideos).to.equal(1)
253 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
254 const stat = await checkStatsGlobal(strategy)
256 expect(stat.totalUsed).to.equal(0)
257 expect(stat.totalVideoFiles).to.equal(0)
258 expect(stat.totalVideos).to.equal(0)
261 async function findServerFollows () {
262 const body = await servers[0].followsCommand.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
263 const follows = body.data
264 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
265 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
267 return { server2, server3 }
270 async function enableRedundancyOnServer1 () {
271 await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
273 const { server2, server3 } = await findServerFollows()
275 expect(server3).to.not.be.undefined
276 expect(server3.following.hostRedundancyAllowed).to.be.false
278 expect(server2).to.not.be.undefined
279 expect(server2.following.hostRedundancyAllowed).to.be.true
282 async function disableRedundancyOnServer1 () {
283 await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
285 const { server2, server3 } = await findServerFollows()
287 expect(server3).to.not.be.undefined
288 expect(server3.following.hostRedundancyAllowed).to.be.false
290 expect(server2).to.not.be.undefined
291 expect(server2.following.hostRedundancyAllowed).to.be.false
294 describe('Test videos redundancy', function () {
296 describe('With most-views strategy', function () {
297 const strategy = 'most-views'
302 return flushAndRunServers(strategy)
305 it('Should have 1 webseed on the first video', async function () {
306 await check1WebSeed()
307 await check0PlaylistRedundancies()
308 await checkStatsWithoutRedundancy(strategy)
311 it('Should enable redundancy on server 1', function () {
312 return enableRedundancyOnServer1()
315 it('Should have 2 webseeds on the first video', async function () {
318 await waitJobs(servers)
319 await waitUntilLog(servers[0], 'Duplicated ', 5)
320 await waitJobs(servers)
322 await check2Webseeds()
323 await check1PlaylistRedundancies()
324 await checkStatsWith1Redundancy(strategy)
327 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
330 await disableRedundancyOnServer1()
332 await waitJobs(servers)
335 await check1WebSeed()
336 await check0PlaylistRedundancies()
338 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
341 after(async function () {
342 return cleanupTests(servers)
346 describe('With trending strategy', function () {
347 const strategy = 'trending'
352 return flushAndRunServers(strategy)
355 it('Should have 1 webseed on the first video', async function () {
356 await check1WebSeed()
357 await check0PlaylistRedundancies()
358 await checkStatsWithoutRedundancy(strategy)
361 it('Should enable redundancy on server 1', function () {
362 return enableRedundancyOnServer1()
365 it('Should have 2 webseeds on the first video', async function () {
368 await waitJobs(servers)
369 await waitUntilLog(servers[0], 'Duplicated ', 5)
370 await waitJobs(servers)
372 await check2Webseeds()
373 await check1PlaylistRedundancies()
374 await checkStatsWith1Redundancy(strategy)
377 it('Should unfollow on server 1 and remove duplicated videos', async function () {
380 await servers[0].followsCommand.unfollow({ target: servers[1] })
382 await waitJobs(servers)
385 await check1WebSeed()
386 await check0PlaylistRedundancies()
388 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
391 after(async function () {
392 await cleanupTests(servers)
396 describe('With recently added strategy', function () {
397 const strategy = 'recently-added'
402 return flushAndRunServers(strategy, { min_views: 3 })
405 it('Should have 1 webseed on the first video', async function () {
406 await check1WebSeed()
407 await check0PlaylistRedundancies()
408 await checkStatsWithoutRedundancy(strategy)
411 it('Should enable redundancy on server 1', function () {
412 return enableRedundancyOnServer1()
415 it('Should still have 1 webseed on the first video', async function () {
418 await waitJobs(servers)
420 await waitJobs(servers)
422 await check1WebSeed()
423 await check0PlaylistRedundancies()
424 await checkStatsWithoutRedundancy(strategy)
427 it('Should view 2 times the first video to have > min_views config', async function () {
430 await viewVideo(servers[0].url, video1Server2UUID)
431 await viewVideo(servers[2].url, video1Server2UUID)
434 await waitJobs(servers)
437 it('Should have 2 webseeds on the first video', async function () {
440 await waitJobs(servers)
441 await waitUntilLog(servers[0], 'Duplicated ', 5)
442 await waitJobs(servers)
444 await check2Webseeds()
445 await check1PlaylistRedundancies()
446 await checkStatsWith1Redundancy(strategy)
449 it('Should remove the video and the redundancy files', async function () {
452 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
454 await waitJobs(servers)
456 for (const server of servers) {
457 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
461 after(async function () {
462 await cleanupTests(servers)
466 describe('With only HLS files', function () {
467 const strategy = 'recently-added'
469 before(async function () {
472 await flushAndRunServers(strategy, { min_views: 3 }, false)
475 it('Should have 0 playlist redundancy on the first video', async function () {
476 await check1WebSeed()
477 await check0PlaylistRedundancies()
480 it('Should enable redundancy on server 1', function () {
481 return enableRedundancyOnServer1()
484 it('Should still have 0 redundancy on the first video', async function () {
487 await waitJobs(servers)
489 await waitJobs(servers)
491 await check0PlaylistRedundancies()
492 await checkStatsWithoutRedundancy(strategy)
495 it('Should have 1 redundancy on the first video', async function () {
498 await viewVideo(servers[0].url, video1Server2UUID)
499 await viewVideo(servers[2].url, video1Server2UUID)
502 await waitJobs(servers)
504 await waitJobs(servers)
505 await waitUntilLog(servers[0], 'Duplicated ', 1)
506 await waitJobs(servers)
508 await check1PlaylistRedundancies()
509 await checkStatsWith1Redundancy(strategy, true)
512 it('Should remove the video and the redundancy files', async function () {
515 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
517 await waitJobs(servers)
519 for (const server of servers) {
520 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
524 after(async function () {
525 await cleanupTests(servers)
529 describe('With manual strategy', function () {
533 return flushAndRunServers(null)
536 it('Should have 1 webseed on the first video', async function () {
537 await check1WebSeed()
538 await check0PlaylistRedundancies()
539 await checkStatsWithoutRedundancy('manual')
542 it('Should create a redundancy on first video', async function () {
543 await servers[0].redundancyCommand.addVideo({ videoId: video1Server2Id })
546 it('Should have 2 webseeds on the first video', async function () {
549 await waitJobs(servers)
550 await waitUntilLog(servers[0], 'Duplicated ', 5)
551 await waitJobs(servers)
553 await check2Webseeds()
554 await check1PlaylistRedundancies()
555 await checkStatsWith1Redundancy('manual')
558 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
561 const body = await servers[0].redundancyCommand.listVideos({ target: 'remote-videos' })
563 const videos = body.data
564 expect(videos).to.have.lengthOf(1)
566 const video = videos[0]
568 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
569 await servers[0].redundancyCommand.removeVideo({ redundancyId: r.id })
572 await waitJobs(servers)
575 await check1WebSeed()
576 await check0PlaylistRedundancies()
578 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
581 after(async function () {
582 await cleanupTests(servers)
586 describe('Test expiration', function () {
587 const strategy = 'recently-added'
589 async function checkContains (servers: ServerInfo[], str: string) {
590 for (const server of servers) {
591 const res = await getVideo(server.url, video1Server2UUID)
592 const video: VideoDetails = res.body
594 for (const f of video.files) {
595 expect(f.magnetUri).to.contain(str)
600 async function checkNotContains (servers: ServerInfo[], str: string) {
601 for (const server of servers) {
602 const res = await getVideo(server.url, video1Server2UUID)
603 const video: VideoDetails = res.body
605 for (const f of video.files) {
606 expect(f.magnetUri).to.not.contain(str)
611 before(async function () {
614 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
616 await enableRedundancyOnServer1()
619 it('Should still have 2 webseeds after 10 seconds', async function () {
625 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
627 // Maybe a server deleted a redundancy in the scheduler
630 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
634 it('Should stop server 1 and expire video redundancy', async function () {
637 killallServers([ servers[0] ])
641 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
644 after(async function () {
645 await cleanupTests(servers)
649 describe('Test file replacement', function () {
650 let video2Server2UUID: string
651 const strategy = 'recently-added'
653 before(async function () {
656 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
658 await enableRedundancyOnServer1()
660 await waitJobs(servers)
661 await waitUntilLog(servers[0], 'Duplicated ', 5)
662 await waitJobs(servers)
664 await check2Webseeds(video1Server2UUID)
665 await check1PlaylistRedundancies(video1Server2UUID)
666 await checkStatsWith1Redundancy(strategy)
668 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE })
669 video2Server2UUID = res.body.video.uuid
671 // Wait transcoding before federation
672 await waitJobs(servers)
674 await updateVideo(servers[1].url, servers[1].accessToken, video2Server2UUID, { privacy: VideoPrivacy.PUBLIC })
677 it('Should cache video 2 webseeds on the first video', async function () {
680 await waitJobs(servers)
684 while (checked === false) {
688 await check1WebSeed(video1Server2UUID)
689 await check0PlaylistRedundancies(video1Server2UUID)
691 await check2Webseeds(video2Server2UUID)
692 await check1PlaylistRedundancies(video2Server2UUID)
701 it('Should disable strategy and remove redundancies', async function () {
704 await waitJobs(servers)
706 killallServers([ servers[0] ])
707 await reRunServer(servers[0], {
710 check_interval: '1 second',
716 await waitJobs(servers)
718 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
721 after(async function () {
722 await cleanupTests(servers)