1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { readdir } from 'fs-extra'
6 import * as magnetUtil from 'magnet-uri'
7 import { join } from 'path'
8 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
11 checkVideoFilesWereRemoved,
14 flushAndRunMultipleServers,
24 setAccessTokensToServers,
30 } from '../../../../shared/extra-utils'
31 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
34 listVideoRedundancies,
35 removeVideoRedundancy,
37 } from '../../../../shared/extra-utils/server/redundancy'
38 import { getStats } from '../../../../shared/extra-utils/server/stats'
39 import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
40 import { ServerStats } from '../../../../shared/models/server/server-stats.model'
41 import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos'
43 const expect = chai.expect
45 let servers: ServerInfo[] = []
46 let video1Server2UUID: string
47 let video1Server2Id: number
49 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
50 const parsed = magnetUtil.decode(file.magnetUri)
52 for (const ws of baseWebseeds) {
53 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
54 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
57 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
60 async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
61 const strategies: any[] = []
63 if (strategy !== null) {
66 min_lifetime: '1 hour',
76 enabled: withWebtorrent
84 check_interval: '5 seconds',
90 servers = await flushAndRunMultipleServers(3, config)
92 // Get the access tokens
93 await setAccessTokensToServers(servers)
96 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
97 video1Server2UUID = res.body.video.uuid
98 video1Server2Id = res.body.video.id
100 await viewVideo(servers[1].url, video1Server2UUID)
103 await waitJobs(servers)
105 // Server 1 and server 2 follow each other
106 await doubleFollow(servers[0], servers[1])
107 // Server 1 and server 3 follow each other
108 await doubleFollow(servers[0], servers[2])
109 // Server 2 and server 3 follow each other
110 await doubleFollow(servers[1], servers[2])
112 await waitJobs(servers)
115 async function check1WebSeed (videoUUID?: string) {
116 if (!videoUUID) videoUUID = video1Server2UUID
119 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
122 for (const server of servers) {
123 // With token to avoid issues with video follow constraints
124 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
126 const video: VideoDetails = res.body
127 for (const f of video.files) {
128 checkMagnetWebseeds(f, webseeds, server)
133 async function check2Webseeds (videoUUID?: string) {
134 if (!videoUUID) videoUUID = video1Server2UUID
137 `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`,
138 `http://localhost:${servers[1].port}/static/webseed/${videoUUID}`
141 for (const server of servers) {
142 const res = await getVideo(server.url, videoUUID)
144 const video: VideoDetails = res.body
146 for (const file of video.files) {
147 checkMagnetWebseeds(file, webseeds, server)
149 await makeGetRequest({
151 statusCodeExpected: HttpStatusCode.OK_200,
152 path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
155 await makeGetRequest({
157 statusCodeExpected: HttpStatusCode.OK_200,
158 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
164 const directories = [
165 'test' + servers[0].internalServerNumber + '/redundancy',
166 'test' + servers[1].internalServerNumber + '/videos'
169 for (const directory of directories) {
170 const files = await readdir(join(root(), directory))
171 expect(files).to.have.length.at.least(4)
173 for (const resolution of [ 240, 360, 480, 720 ]) {
174 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
179 async function check0PlaylistRedundancies (videoUUID?: string) {
180 if (!videoUUID) videoUUID = video1Server2UUID
182 for (const server of servers) {
183 // With token to avoid issues with video follow constraints
184 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
185 const video: VideoDetails = res.body
187 expect(video.streamingPlaylists).to.be.an('array')
188 expect(video.streamingPlaylists).to.have.lengthOf(1)
189 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
193 async function check1PlaylistRedundancies (videoUUID?: string) {
194 if (!videoUUID) videoUUID = video1Server2UUID
196 for (const server of servers) {
197 const res = await getVideo(server.url, videoUUID)
198 const video: VideoDetails = res.body
200 expect(video.streamingPlaylists).to.have.lengthOf(1)
201 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
203 const redundancy = video.streamingPlaylists[0].redundancies[0]
205 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
208 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls'
209 const baseUrlSegment = servers[0].url + '/static/redundancy/hls'
211 const res = await getVideo(servers[0].url, videoUUID)
212 const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0]
214 for (const resolution of [ 240, 360, 480, 720 ]) {
215 await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist)
218 const directories = [
219 'test' + servers[0].internalServerNumber + '/redundancy/hls',
220 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
223 for (const directory of directories) {
224 const files = await readdir(join(root(), directory, videoUUID))
225 expect(files).to.have.length.at.least(4)
227 for (const resolution of [ 240, 360, 480, 720 ]) {
228 const filename = `${videoUUID}-${resolution}-fragmented.mp4`
230 expect(files.find(f => f === filename)).to.not.be.undefined
235 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
236 let totalSize: number = null
239 if (strategy !== 'manual') {
244 const res = await getStats(servers[0].url)
245 const data: ServerStats = res.body
247 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
249 const stat = data.videosRedundancy[0]
250 expect(stat.strategy).to.equal(strategy)
251 expect(stat.totalSize).to.equal(totalSize)
256 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
257 const stat = await checkStatsGlobal(strategy)
259 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
260 expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
261 expect(stat.totalVideos).to.equal(1)
264 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
265 const stat = await checkStatsGlobal(strategy)
267 expect(stat.totalUsed).to.equal(0)
268 expect(stat.totalVideoFiles).to.equal(0)
269 expect(stat.totalVideos).to.equal(0)
272 async function findServerFollows () {
273 const body = await servers[0].followsCommand.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
274 const follows = body.data
275 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
276 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
278 return { server2, server3 }
281 async function enableRedundancyOnServer1 () {
282 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
284 const { server2, server3 } = await findServerFollows()
286 expect(server3).to.not.be.undefined
287 expect(server3.following.hostRedundancyAllowed).to.be.false
289 expect(server2).to.not.be.undefined
290 expect(server2.following.hostRedundancyAllowed).to.be.true
293 async function disableRedundancyOnServer1 () {
294 await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
296 const { server2, server3 } = await findServerFollows()
298 expect(server3).to.not.be.undefined
299 expect(server3.following.hostRedundancyAllowed).to.be.false
301 expect(server2).to.not.be.undefined
302 expect(server2.following.hostRedundancyAllowed).to.be.false
305 describe('Test videos redundancy', function () {
307 describe('With most-views strategy', function () {
308 const strategy = 'most-views'
313 return flushAndRunServers(strategy)
316 it('Should have 1 webseed on the first video', async function () {
317 await check1WebSeed()
318 await check0PlaylistRedundancies()
319 await checkStatsWithoutRedundancy(strategy)
322 it('Should enable redundancy on server 1', function () {
323 return enableRedundancyOnServer1()
326 it('Should have 2 webseeds on the first video', async function () {
329 await waitJobs(servers)
330 await waitUntilLog(servers[0], 'Duplicated ', 5)
331 await waitJobs(servers)
333 await check2Webseeds()
334 await check1PlaylistRedundancies()
335 await checkStatsWith1Redundancy(strategy)
338 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
341 await disableRedundancyOnServer1()
343 await waitJobs(servers)
346 await check1WebSeed()
347 await check0PlaylistRedundancies()
349 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos', join('playlists', 'hls') ])
352 after(async function () {
353 return cleanupTests(servers)
357 describe('With trending strategy', function () {
358 const strategy = 'trending'
363 return flushAndRunServers(strategy)
366 it('Should have 1 webseed on the first video', async function () {
367 await check1WebSeed()
368 await check0PlaylistRedundancies()
369 await checkStatsWithoutRedundancy(strategy)
372 it('Should enable redundancy on server 1', function () {
373 return enableRedundancyOnServer1()
376 it('Should have 2 webseeds on the first video', async function () {
379 await waitJobs(servers)
380 await waitUntilLog(servers[0], 'Duplicated ', 5)
381 await waitJobs(servers)
383 await check2Webseeds()
384 await check1PlaylistRedundancies()
385 await checkStatsWith1Redundancy(strategy)
388 it('Should unfollow on server 1 and remove duplicated videos', async function () {
391 await servers[0].followsCommand.unfollow({ target: servers[1] })
393 await waitJobs(servers)
396 await check1WebSeed()
397 await check0PlaylistRedundancies()
399 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ 'videos' ])
402 after(async function () {
403 await cleanupTests(servers)
407 describe('With recently added strategy', function () {
408 const strategy = 'recently-added'
413 return flushAndRunServers(strategy, { min_views: 3 })
416 it('Should have 1 webseed on the first video', async function () {
417 await check1WebSeed()
418 await check0PlaylistRedundancies()
419 await checkStatsWithoutRedundancy(strategy)
422 it('Should enable redundancy on server 1', function () {
423 return enableRedundancyOnServer1()
426 it('Should still have 1 webseed on the first video', async function () {
429 await waitJobs(servers)
431 await waitJobs(servers)
433 await check1WebSeed()
434 await check0PlaylistRedundancies()
435 await checkStatsWithoutRedundancy(strategy)
438 it('Should view 2 times the first video to have > min_views config', async function () {
441 await viewVideo(servers[0].url, video1Server2UUID)
442 await viewVideo(servers[2].url, video1Server2UUID)
445 await waitJobs(servers)
448 it('Should have 2 webseeds on the first video', async function () {
451 await waitJobs(servers)
452 await waitUntilLog(servers[0], 'Duplicated ', 5)
453 await waitJobs(servers)
455 await check2Webseeds()
456 await check1PlaylistRedundancies()
457 await checkStatsWith1Redundancy(strategy)
460 it('Should remove the video and the redundancy files', async function () {
463 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
465 await waitJobs(servers)
467 for (const server of servers) {
468 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
472 after(async function () {
473 await cleanupTests(servers)
477 describe('With only HLS files', function () {
478 const strategy = 'recently-added'
480 before(async function () {
483 await flushAndRunServers(strategy, { min_views: 3 }, false)
486 it('Should have 0 playlist redundancy on the first video', async function () {
487 await check1WebSeed()
488 await check0PlaylistRedundancies()
491 it('Should enable redundancy on server 1', function () {
492 return enableRedundancyOnServer1()
495 it('Should still have 0 redundancy on the first video', async function () {
498 await waitJobs(servers)
500 await waitJobs(servers)
502 await check0PlaylistRedundancies()
503 await checkStatsWithoutRedundancy(strategy)
506 it('Should have 1 redundancy on the first video', async function () {
509 await viewVideo(servers[0].url, video1Server2UUID)
510 await viewVideo(servers[2].url, video1Server2UUID)
513 await waitJobs(servers)
515 await waitJobs(servers)
516 await waitUntilLog(servers[0], 'Duplicated ', 1)
517 await waitJobs(servers)
519 await check1PlaylistRedundancies()
520 await checkStatsWith1Redundancy(strategy, true)
523 it('Should remove the video and the redundancy files', async function () {
526 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
528 await waitJobs(servers)
530 for (const server of servers) {
531 await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
535 after(async function () {
536 await cleanupTests(servers)
540 describe('With manual strategy', function () {
544 return flushAndRunServers(null)
547 it('Should have 1 webseed on the first video', async function () {
548 await check1WebSeed()
549 await check0PlaylistRedundancies()
550 await checkStatsWithoutRedundancy('manual')
553 it('Should create a redundancy on first video', async function () {
554 await addVideoRedundancy({
556 accessToken: servers[0].accessToken,
557 videoId: video1Server2Id
561 it('Should have 2 webseeds on the first video', async function () {
564 await waitJobs(servers)
565 await waitUntilLog(servers[0], 'Duplicated ', 5)
566 await waitJobs(servers)
568 await check2Webseeds()
569 await check1PlaylistRedundancies()
570 await checkStatsWith1Redundancy('manual')
573 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
576 const res = await listVideoRedundancies({
578 accessToken: servers[0].accessToken,
579 target: 'remote-videos'
582 const videos = res.body.data as VideoRedundancy[]
583 expect(videos).to.have.lengthOf(1)
585 const video = videos[0]
586 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
587 await removeVideoRedundancy({
589 accessToken: servers[0].accessToken,
594 await waitJobs(servers)
597 await check1WebSeed()
598 await check0PlaylistRedundancies()
600 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
603 after(async function () {
604 await cleanupTests(servers)
608 describe('Test expiration', function () {
609 const strategy = 'recently-added'
611 async function checkContains (servers: ServerInfo[], str: string) {
612 for (const server of servers) {
613 const res = await getVideo(server.url, video1Server2UUID)
614 const video: VideoDetails = res.body
616 for (const f of video.files) {
617 expect(f.magnetUri).to.contain(str)
622 async function checkNotContains (servers: ServerInfo[], str: string) {
623 for (const server of servers) {
624 const res = await getVideo(server.url, video1Server2UUID)
625 const video: VideoDetails = res.body
627 for (const f of video.files) {
628 expect(f.magnetUri).to.not.contain(str)
633 before(async function () {
636 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
638 await enableRedundancyOnServer1()
641 it('Should still have 2 webseeds after 10 seconds', async function () {
647 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
649 // Maybe a server deleted a redundancy in the scheduler
652 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
656 it('Should stop server 1 and expire video redundancy', async function () {
659 killallServers([ servers[0] ])
663 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
666 after(async function () {
667 await cleanupTests(servers)
671 describe('Test file replacement', function () {
672 let video2Server2UUID: string
673 const strategy = 'recently-added'
675 before(async function () {
678 await flushAndRunServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
680 await enableRedundancyOnServer1()
682 await waitJobs(servers)
683 await waitUntilLog(servers[0], 'Duplicated ', 5)
684 await waitJobs(servers)
686 await check2Webseeds(video1Server2UUID)
687 await check1PlaylistRedundancies(video1Server2UUID)
688 await checkStatsWith1Redundancy(strategy)
690 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE })
691 video2Server2UUID = res.body.video.uuid
693 // Wait transcoding before federation
694 await waitJobs(servers)
696 await updateVideo(servers[1].url, servers[1].accessToken, video2Server2UUID, { privacy: VideoPrivacy.PUBLIC })
699 it('Should cache video 2 webseeds on the first video', async function () {
702 await waitJobs(servers)
706 while (checked === false) {
710 await check1WebSeed(video1Server2UUID)
711 await check0PlaylistRedundancies(video1Server2UUID)
713 await check2Webseeds(video2Server2UUID)
714 await check1PlaylistRedundancies(video2Server2UUID)
723 it('Should disable strategy and remove redundancies', async function () {
726 await waitJobs(servers)
728 killallServers([ servers[0] ])
729 await reRunServer(servers[0], {
732 check_interval: '1 second',
738 await waitJobs(servers)
740 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].internalServerNumber, [ join('redundancy', 'hls') ])
743 after(async function () {
744 await cleanupTests(servers)