1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { readdir } from 'fs-extra'
6 import magnetUtil from 'magnet-uri'
7 import { basename, join } from 'path'
10 checkVideoFilesWereRemoved,
12 createMultipleServers,
19 setAccessTokensToServers,
22 } from '@shared/extra-utils'
28 VideoRedundancyStrategy,
29 VideoRedundancyStrategyWithManual
30 } from '@shared/models'
32 const expect = chai.expect
34 let servers: PeerTubeServer[] = []
35 let video1Server2: VideoDetails
37 async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) {
38 const parsed = magnetUtil.decode(file.magnetUri)
40 for (const ws of baseWebseeds) {
41 const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`)
42 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
45 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
47 for (const url of parsed.urlList) {
48 await makeRawRequest(url, HttpStatusCode.OK_200)
52 async function createServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
53 const strategies: any[] = []
55 if (strategy !== null) {
58 min_lifetime: '1 hour',
70 enabled: withWebtorrent
78 check_interval: '5 seconds',
84 servers = await createMultipleServers(3, config)
86 // Get the access tokens
87 await setAccessTokensToServers(servers)
90 const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } })
91 video1Server2 = await servers[1].videos.get({ id })
93 await servers[1].videos.view({ id })
96 await waitJobs(servers)
98 // Server 1 and server 2 follow each other
99 await doubleFollow(servers[0], servers[1])
100 // Server 1 and server 3 follow each other
101 await doubleFollow(servers[0], servers[2])
102 // Server 2 and server 3 follow each other
103 await doubleFollow(servers[1], servers[2])
105 await waitJobs(servers)
108 async function ensureSameFilenames (videoUUID: string) {
109 let webtorrentFilenames: string[]
110 let hlsFilenames: string[]
112 for (const server of servers) {
113 const video = await server.videos.getWithToken({ id: videoUUID })
115 // Ensure we use the same filenames that the origin
117 const localWebtorrentFilenames = video.files.map(f => basename(f.fileUrl)).sort()
118 const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort()
120 if (webtorrentFilenames) expect(webtorrentFilenames).to.deep.equal(localWebtorrentFilenames)
121 else webtorrentFilenames = localWebtorrentFilenames
123 if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames)
124 else hlsFilenames = localHLSFilenames
127 return { webtorrentFilenames, hlsFilenames }
130 async function check1WebSeed (videoUUID?: string) {
131 if (!videoUUID) videoUUID = video1Server2.uuid
134 `http://localhost:${servers[1].port}/static/webseed/`
137 for (const server of servers) {
138 // With token to avoid issues with video follow constraints
139 const video = await server.videos.getWithToken({ id: videoUUID })
141 for (const f of video.files) {
142 await checkMagnetWebseeds(f, webseeds, server)
146 await ensureSameFilenames(videoUUID)
149 async function check2Webseeds (videoUUID?: string) {
150 if (!videoUUID) videoUUID = video1Server2.uuid
153 `http://localhost:${servers[0].port}/static/redundancy/`,
154 `http://localhost:${servers[1].port}/static/webseed/`
157 for (const server of servers) {
158 const video = await server.videos.get({ id: videoUUID })
160 for (const file of video.files) {
161 await checkMagnetWebseeds(file, webseeds, server)
165 const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
167 const directories = [
168 'test' + servers[0].internalServerNumber + '/redundancy',
169 'test' + servers[1].internalServerNumber + '/videos'
172 for (const directory of directories) {
173 const files = await readdir(join(root(), directory))
174 expect(files).to.have.length.at.least(4)
176 // Ensure we files exist on disk
177 expect(files.find(f => webtorrentFilenames.includes(f))).to.exist
181 async function check0PlaylistRedundancies (videoUUID?: string) {
182 if (!videoUUID) videoUUID = video1Server2.uuid
184 for (const server of servers) {
185 // With token to avoid issues with video follow constraints
186 const video = await server.videos.getWithToken({ id: videoUUID })
188 expect(video.streamingPlaylists).to.be.an('array')
189 expect(video.streamingPlaylists).to.have.lengthOf(1)
190 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
193 await ensureSameFilenames(videoUUID)
196 async function check1PlaylistRedundancies (videoUUID?: string) {
197 if (!videoUUID) videoUUID = video1Server2.uuid
199 for (const server of servers) {
200 const video = await server.videos.get({ id: videoUUID })
202 expect(video.streamingPlaylists).to.have.lengthOf(1)
203 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
205 const redundancy = video.streamingPlaylists[0].redundancies[0]
207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID
211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID
213 const video = await servers[0].videos.get({ id: videoUUID })
214 const hlsPlaylist = video.streamingPlaylists[0]
216 for (const resolution of [ 240, 360, 480, 720 ]) {
217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist })
220 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
222 const directories = [
223 'test' + servers[0].internalServerNumber + '/redundancy/hls',
224 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
227 for (const directory of directories) {
228 const files = await readdir(join(root(), directory, videoUUID))
229 expect(files).to.have.length.at.least(4)
231 // Ensure we files exist on disk
232 expect(files.find(f => hlsFilenames.includes(f))).to.exist
236 async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
237 let totalSize: number = null
240 if (strategy !== 'manual') {
245 const data = await servers[0].stats.get()
246 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
248 const stat = data.videosRedundancy[0]
249 expect(stat.strategy).to.equal(strategy)
250 expect(stat.totalSize).to.equal(totalSize)
255 async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) {
256 const stat = await checkStatsGlobal(strategy)
258 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
259 expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8)
260 expect(stat.totalVideos).to.equal(1)
263 async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
264 const stat = await checkStatsGlobal(strategy)
266 expect(stat.totalUsed).to.equal(0)
267 expect(stat.totalVideoFiles).to.equal(0)
268 expect(stat.totalVideos).to.equal(0)
271 async function findServerFollows () {
272 const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' })
273 const follows = body.data
274 const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`)
275 const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`)
277 return { server2, server3 }
280 async function enableRedundancyOnServer1 () {
281 await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
283 const { server2, server3 } = await findServerFollows()
285 expect(server3).to.not.be.undefined
286 expect(server3.following.hostRedundancyAllowed).to.be.false
288 expect(server2).to.not.be.undefined
289 expect(server2.following.hostRedundancyAllowed).to.be.true
292 async function disableRedundancyOnServer1 () {
293 await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
295 const { server2, server3 } = await findServerFollows()
297 expect(server3).to.not.be.undefined
298 expect(server3.following.hostRedundancyAllowed).to.be.false
300 expect(server2).to.not.be.undefined
301 expect(server2.following.hostRedundancyAllowed).to.be.false
304 describe('Test videos redundancy', function () {
306 describe('With most-views strategy', function () {
307 const strategy = 'most-views'
312 return createServers(strategy)
315 it('Should have 1 webseed on the first video', async function () {
316 await check1WebSeed()
317 await check0PlaylistRedundancies()
318 await checkStatsWithoutRedundancy(strategy)
321 it('Should enable redundancy on server 1', function () {
322 return enableRedundancyOnServer1()
325 it('Should have 2 webseeds on the first video', async function () {
328 await waitJobs(servers)
329 await servers[0].servers.waitUntilLog('Duplicated ', 5)
330 await waitJobs(servers)
332 await check2Webseeds()
333 await check1PlaylistRedundancies()
334 await checkStatsWith1Redundancy(strategy)
337 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
340 await disableRedundancyOnServer1()
342 await waitJobs(servers)
345 await check1WebSeed()
346 await check0PlaylistRedundancies()
348 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
351 after(async function () {
352 return cleanupTests(servers)
356 describe('With trending strategy', function () {
357 const strategy = 'trending'
362 return createServers(strategy)
365 it('Should have 1 webseed on the first video', async function () {
366 await check1WebSeed()
367 await check0PlaylistRedundancies()
368 await checkStatsWithoutRedundancy(strategy)
371 it('Should enable redundancy on server 1', function () {
372 return enableRedundancyOnServer1()
375 it('Should have 2 webseeds on the first video', async function () {
378 await waitJobs(servers)
379 await servers[0].servers.waitUntilLog('Duplicated ', 5)
380 await waitJobs(servers)
382 await check2Webseeds()
383 await check1PlaylistRedundancies()
384 await checkStatsWith1Redundancy(strategy)
387 it('Should unfollow server 3 and keep duplicated videos', async function () {
390 await servers[0].follows.unfollow({ target: servers[2] })
392 await waitJobs(servers)
395 await check2Webseeds()
396 await check1PlaylistRedundancies()
397 await checkStatsWith1Redundancy(strategy)
400 it('Should unfollow server 2 and remove duplicated videos', async function () {
403 await servers[0].follows.unfollow({ target: servers[1] })
405 await waitJobs(servers)
408 await check1WebSeed()
409 await check0PlaylistRedundancies()
411 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
414 after(async function () {
415 await cleanupTests(servers)
419 describe('With recently added strategy', function () {
420 const strategy = 'recently-added'
425 return createServers(strategy, { min_views: 3 })
428 it('Should have 1 webseed on the first video', async function () {
429 await check1WebSeed()
430 await check0PlaylistRedundancies()
431 await checkStatsWithoutRedundancy(strategy)
434 it('Should enable redundancy on server 1', function () {
435 return enableRedundancyOnServer1()
438 it('Should still have 1 webseed on the first video', async function () {
441 await waitJobs(servers)
443 await waitJobs(servers)
445 await check1WebSeed()
446 await check0PlaylistRedundancies()
447 await checkStatsWithoutRedundancy(strategy)
450 it('Should view 2 times the first video to have > min_views config', async function () {
453 await servers[0].videos.view({ id: video1Server2.uuid })
454 await servers[2].videos.view({ id: video1Server2.uuid })
457 await waitJobs(servers)
460 it('Should have 2 webseeds on the first video', async function () {
463 await waitJobs(servers)
464 await servers[0].servers.waitUntilLog('Duplicated ', 5)
465 await waitJobs(servers)
467 await check2Webseeds()
468 await check1PlaylistRedundancies()
469 await checkStatsWith1Redundancy(strategy)
472 it('Should remove the video and the redundancy files', async function () {
475 await saveVideoInServers(servers, video1Server2.uuid)
476 await servers[1].videos.remove({ id: video1Server2.uuid })
478 await waitJobs(servers)
480 for (const server of servers) {
481 await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
485 after(async function () {
486 await cleanupTests(servers)
490 describe('With only HLS files', function () {
491 const strategy = 'recently-added'
493 before(async function () {
496 await createServers(strategy, { min_views: 3 }, false)
499 it('Should have 0 playlist redundancy on the first video', async function () {
500 await check1WebSeed()
501 await check0PlaylistRedundancies()
504 it('Should enable redundancy on server 1', function () {
505 return enableRedundancyOnServer1()
508 it('Should still have 0 redundancy on the first video', async function () {
511 await waitJobs(servers)
513 await waitJobs(servers)
515 await check0PlaylistRedundancies()
516 await checkStatsWithoutRedundancy(strategy)
519 it('Should have 1 redundancy on the first video', async function () {
522 await servers[0].videos.view({ id: video1Server2.uuid })
523 await servers[2].videos.view({ id: video1Server2.uuid })
526 await waitJobs(servers)
528 await waitJobs(servers)
529 await servers[0].servers.waitUntilLog('Duplicated ', 1)
530 await waitJobs(servers)
532 await check1PlaylistRedundancies()
533 await checkStatsWith1Redundancy(strategy, true)
536 it('Should remove the video and the redundancy files', async function () {
539 await saveVideoInServers(servers, video1Server2.uuid)
540 await servers[1].videos.remove({ id: video1Server2.uuid })
542 await waitJobs(servers)
544 for (const server of servers) {
545 await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails })
549 after(async function () {
550 await cleanupTests(servers)
554 describe('With manual strategy', function () {
558 return createServers(null)
561 it('Should have 1 webseed on the first video', async function () {
562 await check1WebSeed()
563 await check0PlaylistRedundancies()
564 await checkStatsWithoutRedundancy('manual')
567 it('Should create a redundancy on first video', async function () {
568 await servers[0].redundancy.addVideo({ videoId: video1Server2.id })
571 it('Should have 2 webseeds on the first video', async function () {
574 await waitJobs(servers)
575 await servers[0].servers.waitUntilLog('Duplicated ', 5)
576 await waitJobs(servers)
578 await check2Webseeds()
579 await check1PlaylistRedundancies()
580 await checkStatsWith1Redundancy('manual')
583 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
586 const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' })
588 const videos = body.data
589 expect(videos).to.have.lengthOf(1)
591 const video = videos[0]
593 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
594 await servers[0].redundancy.removeVideo({ redundancyId: r.id })
597 await waitJobs(servers)
600 await check1WebSeed()
601 await check0PlaylistRedundancies()
603 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
606 after(async function () {
607 await cleanupTests(servers)
611 describe('Test expiration', function () {
612 const strategy = 'recently-added'
614 async function checkContains (servers: PeerTubeServer[], str: string) {
615 for (const server of servers) {
616 const video = await server.videos.get({ id: video1Server2.uuid })
618 for (const f of video.files) {
619 expect(f.magnetUri).to.contain(str)
624 async function checkNotContains (servers: PeerTubeServer[], str: string) {
625 for (const server of servers) {
626 const video = await server.videos.get({ id: video1Server2.uuid })
628 for (const f of video.files) {
629 expect(f.magnetUri).to.not.contain(str)
634 before(async function () {
637 await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
639 await enableRedundancyOnServer1()
642 it('Should still have 2 webseeds after 10 seconds', async function () {
648 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
650 // Maybe a server deleted a redundancy in the scheduler
653 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
657 it('Should stop server 1 and expire video redundancy', async function () {
660 await killallServers([ servers[0] ])
664 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A' + servers[0].port)
667 after(async function () {
668 await cleanupTests(servers)
672 describe('Test file replacement', function () {
673 let video2Server2UUID: string
674 const strategy = 'recently-added'
676 before(async function () {
679 await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
681 await enableRedundancyOnServer1()
683 await waitJobs(servers)
684 await servers[0].servers.waitUntilLog('Duplicated ', 5)
685 await waitJobs(servers)
687 await check2Webseeds()
688 await check1PlaylistRedundancies()
689 await checkStatsWith1Redundancy(strategy)
691 const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } })
692 video2Server2UUID = uuid
694 // Wait transcoding before federation
695 await waitJobs(servers)
697 await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
700 it('Should cache video 2 webseeds on the first video', async function () {
703 await waitJobs(servers)
707 while (checked === false) {
711 await check1WebSeed()
712 await check0PlaylistRedundancies()
714 await check2Webseeds(video2Server2UUID)
715 await check1PlaylistRedundancies(video2Server2UUID)
724 it('Should disable strategy and remove redundancies', async function () {
727 await waitJobs(servers)
729 await killallServers([ servers[0] ])
730 await servers[0].run({
733 check_interval: '1 second',
739 await waitJobs(servers)
741 await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true })
744 after(async function () {
745 await cleanupTests(servers)