aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-10-16 10:55:29 +0200
committerChocobozzz <me@florianbigard.com>2018-10-16 16:41:36 +0200
commit26370ce469913f48b1c10b91d68ac8cb67586ce1 (patch)
treee01c9a35a2cc7508694dab9c0f9828f750389ae2
parentfd28a0fc8833c8cfe1017a88fbda141130b2d263 (diff)
downloadPeerTube-26370ce469913f48b1c10b91d68ac8cb67586ce1.tar.gz
PeerTube-26370ce469913f48b1c10b91d68ac8cb67586ce1.tar.zst
PeerTube-26370ce469913f48b1c10b91d68ac8cb67586ce1.zip
Move redundancy in its own travis container
-rw-r--r--.travis.yml1
-rw-r--r--CHANGELOG.md34
-rwxr-xr-xscripts/travis.sh3
-rw-r--r--server/tests/api/index-4.ts1
-rw-r--r--server/tests/api/index.ts1
-rw-r--r--server/tests/api/redundancy/index.ts1
-rw-r--r--server/tests/api/redundancy/redundancy.ts483
-rw-r--r--server/tests/api/server/index.ts1
8 files changed, 524 insertions, 1 deletions
diff --git a/.travis.yml b/.travis.yml
index 7670cb7c0..3a73e4fc0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,6 +42,7 @@ matrix:
42 - env: TEST_SUITE=api-1 42 - env: TEST_SUITE=api-1
43 - env: TEST_SUITE=api-2 43 - env: TEST_SUITE=api-2
44 - env: TEST_SUITE=api-3 44 - env: TEST_SUITE=api-3
45 - env: TEST_SUITE=api-4
45 - env: TEST_SUITE=cli 46 - env: TEST_SUITE=cli
46 - env: TEST_SUITE=lint 47 - env: TEST_SUITE=lint
47 - env: TEST_SUITE=jest 48 - env: TEST_SUITE=jest
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a1363041..d2bd98f4d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,39 @@
1# Changelog 1# Changelog
2 2
3## v1.1.0-alpha.1
4
5We released this alpha version because some admins/users need some moderation tools we implemented in recent weeks.
6This release could contain bugs. Don't expect a stable v1.1.0 until December :)
7
8### Scripts
9
10 * Use DB information from config/production.yaml in upgrade script ([@ldidry](https://github.com/ldidry))
11 * Add REPL script ([@McFlat](https://github.com/mcflat))
12
13### Docker
14
15 * Add search and import settings env settings env variables ([@kaiyou](https://github.com/kaiyou))
16 * Add docker dev image ([@am97](https://github.com/am97))
17
18### Features
19
20 * Automatically resume videos if the user is logged in
21 * Hide automatically the menu when the window is resized ([@BO41](https://github.com/BO41))
22 * Remove confirm modal for JavaScript/CSS injection ([@scanlime](https://github.com/scanlime))
23 * Set bitrate limits for transcoding ([@Nutomic](https://github.com/nutomic))
24 * Add moderation tools in the account page
25 * Add bulk actions in users table (Delete/Ban for now)
26 * Add search filter in admin users table
27 * Add search filter in admin following
28 * Add search filter in admin followers
29 * Add ability to list all local videos
30 * Add ability for users to mute an account or an instance
31 * Add ability for administrators to mute an account or an instance
32 * Rename "News" category to "News & Politics" ([@daker](https://github.com/daker))
33 * Add explicit error message when changing video ownership ([@lucas-dclrcq](https://github.com/lucas-dclrcq))
34 * Improve description of the HTTP video import feature ([@rigelk](https://github.com/rigelk))
35
36
3## v1.0.0 37## v1.0.0
4 38
5### SECURITY 39### SECURITY
diff --git a/scripts/travis.sh b/scripts/travis.sh
index 628039ab7..ae4a9f926 100755
--- a/scripts/travis.sh
+++ b/scripts/travis.sh
@@ -31,6 +31,9 @@ elif [ "$1" = "api-2" ]; then
31elif [ "$1" = "api-3" ]; then 31elif [ "$1" = "api-3" ]; then
32 npm run build:server 32 npm run build:server
33 mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts 33 mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts
34elif [ "$1" = "api-3" ]; then
35 npm run build:server
36 mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-4.ts
34elif [ "$1" = "lint" ]; then 37elif [ "$1" = "lint" ]; then
35 npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts" 38 npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
36 39
diff --git a/server/tests/api/index-4.ts b/server/tests/api/index-4.ts
new file mode 100644
index 000000000..8e69b95a6
--- /dev/null
+++ b/server/tests/api/index-4.ts
@@ -0,0 +1 @@
import './redundancy'
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts
index 2d996dbf9..bc140f860 100644
--- a/server/tests/api/index.ts
+++ b/server/tests/api/index.ts
@@ -2,3 +2,4 @@
2import './index-1' 2import './index-1'
3import './index-2' 3import './index-2'
4import './index-3' 4import './index-3'
5import './index-4'
diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts
new file mode 100644
index 000000000..8e69b95a6
--- /dev/null
+++ b/server/tests/api/redundancy/index.ts
@@ -0,0 +1 @@
import './redundancy'
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
new file mode 100644
index 000000000..1960854b6
--- /dev/null
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -0,0 +1,483 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import { VideoDetails } from '../../../../shared/models/videos'
6import {
7 doubleFollow,
8 flushAndRunMultipleServers,
9 getFollowingListPaginationAndSort,
10 getVideo,
11 immutableAssign,
12 killallServers, makeGetRequest,
13 root,
14 ServerInfo,
15 setAccessTokensToServers, unfollow,
16 uploadVideo,
17 viewVideo,
18 wait,
19 waitUntilLog,
20 checkVideoFilesWereRemoved, removeVideo
21} from '../../utils'
22import { waitJobs } from '../../utils/server/jobs'
23import * as magnetUtil from 'magnet-uri'
24import { updateRedundancy } from '../../utils/server/redundancy'
25import { ActorFollow } from '../../../../shared/models/actors'
26import { readdir } from 'fs-extra'
27import { join } from 'path'
28import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
29import { getStats } from '../../utils/server/stats'
30import { ServerStats } from '../../../../shared/models/server/server-stats.model'
31
32const expect = chai.expect
33
34let servers: ServerInfo[] = []
35let video1Server2UUID: string
36
37function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
38 const parsed = magnetUtil.decode(file.magnetUri)
39
40 for (const ws of baseWebseeds) {
41 const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
42 expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
43 }
44
45 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
46}
47
48async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
49 const config = {
50 redundancy: {
51 videos: {
52 check_interval: '5 seconds',
53 strategies: [
54 immutableAssign({
55 min_lifetime: '1 hour',
56 strategy: strategy,
57 size: '100KB'
58 }, additionalParams)
59 ]
60 }
61 }
62 }
63 servers = await flushAndRunMultipleServers(3, config)
64
65 // Get the access tokens
66 await setAccessTokensToServers(servers)
67
68 {
69 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
70 video1Server2UUID = res.body.video.uuid
71
72 await viewVideo(servers[ 1 ].url, video1Server2UUID)
73 }
74
75 await waitJobs(servers)
76
77 // Server 1 and server 2 follow each other
78 await doubleFollow(servers[ 0 ], servers[ 1 ])
79 // Server 1 and server 3 follow each other
80 await doubleFollow(servers[ 0 ], servers[ 2 ])
81 // Server 2 and server 3 follow each other
82 await doubleFollow(servers[ 1 ], servers[ 2 ])
83
84 await waitJobs(servers)
85}
86
87async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
88 if (!videoUUID) videoUUID = video1Server2UUID
89
90 const webseeds = [
91 'http://localhost:9002/static/webseed/' + videoUUID
92 ]
93
94 for (const server of servers) {
95 {
96 const res = await getVideo(server.url, videoUUID)
97
98 const video: VideoDetails = res.body
99 for (const f of video.files) {
100 checkMagnetWebseeds(f, webseeds, server)
101 }
102 }
103 }
104}
105
106async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
107 const res = await getStats(servers[0].url)
108 const data: ServerStats = res.body
109
110 expect(data.videosRedundancy).to.have.lengthOf(1)
111 const stat = data.videosRedundancy[0]
112
113 expect(stat.strategy).to.equal(strategy)
114 expect(stat.totalSize).to.equal(102400)
115 expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
116 expect(stat.totalVideoFiles).to.equal(4)
117 expect(stat.totalVideos).to.equal(1)
118}
119
120async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
121 const res = await getStats(servers[0].url)
122 const data: ServerStats = res.body
123
124 expect(data.videosRedundancy).to.have.lengthOf(1)
125
126 const stat = data.videosRedundancy[0]
127 expect(stat.strategy).to.equal(strategy)
128 expect(stat.totalSize).to.equal(102400)
129 expect(stat.totalUsed).to.equal(0)
130 expect(stat.totalVideoFiles).to.equal(0)
131 expect(stat.totalVideos).to.equal(0)
132}
133
134async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
135 if (!videoUUID) videoUUID = video1Server2UUID
136
137 const webseeds = [
138 'http://localhost:9001/static/webseed/' + videoUUID,
139 'http://localhost:9002/static/webseed/' + videoUUID
140 ]
141
142 for (const server of servers) {
143 const res = await getVideo(server.url, videoUUID)
144
145 const video: VideoDetails = res.body
146
147 for (const file of video.files) {
148 checkMagnetWebseeds(file, webseeds, server)
149
150 // Only servers 1 and 2 have the video
151 if (server.serverNumber !== 3) {
152 await makeGetRequest({
153 url: server.url,
154 statusCodeExpected: 200,
155 path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
156 contentType: null
157 })
158 }
159 }
160 }
161
162 for (const directory of [ 'test1', 'test2' ]) {
163 const files = await readdir(join(root(), directory, 'videos'))
164 expect(files).to.have.length.at.least(4)
165
166 for (const resolution of [ 240, 360, 480, 720 ]) {
167 expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
168 }
169 }
170}
171
172async function enableRedundancyOnServer1 () {
173 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
174
175 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
176 const follows: ActorFollow[] = res.body.data
177 const server2 = follows.find(f => f.following.host === 'localhost:9002')
178 const server3 = follows.find(f => f.following.host === 'localhost:9003')
179
180 expect(server3).to.not.be.undefined
181 expect(server3.following.hostRedundancyAllowed).to.be.false
182
183 expect(server2).to.not.be.undefined
184 expect(server2.following.hostRedundancyAllowed).to.be.true
185}
186
187async function disableRedundancyOnServer1 () {
188 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
189
190 const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
191 const follows: ActorFollow[] = res.body.data
192 const server2 = follows.find(f => f.following.host === 'localhost:9002')
193 const server3 = follows.find(f => f.following.host === 'localhost:9003')
194
195 expect(server3).to.not.be.undefined
196 expect(server3.following.hostRedundancyAllowed).to.be.false
197
198 expect(server2).to.not.be.undefined
199 expect(server2.following.hostRedundancyAllowed).to.be.false
200}
201
202async function cleanServers () {
203 killallServers(servers)
204}
205
206describe('Test videos redundancy', function () {
207
208 describe('With most-views strategy', function () {
209 const strategy = 'most-views'
210
211 before(function () {
212 this.timeout(120000)
213
214 return runServers(strategy)
215 })
216
217 it('Should have 1 webseed on the first video', async function () {
218 await check1WebSeed(strategy)
219 await checkStatsWith1Webseed(strategy)
220 })
221
222 it('Should enable redundancy on server 1', function () {
223 return enableRedundancyOnServer1()
224 })
225
226 it('Should have 2 webseed on the first video', async function () {
227 this.timeout(40000)
228
229 await waitJobs(servers)
230 await waitUntilLog(servers[0], 'Duplicated ', 4)
231 await waitJobs(servers)
232
233 await check2Webseeds(strategy)
234 await checkStatsWith2Webseed(strategy)
235 })
236
237 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
238 this.timeout(40000)
239
240 await disableRedundancyOnServer1()
241
242 await waitJobs(servers)
243 await wait(5000)
244
245 await check1WebSeed(strategy)
246
247 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
248 })
249
250 after(function () {
251 return cleanServers()
252 })
253 })
254
255 describe('With trending strategy', function () {
256 const strategy = 'trending'
257
258 before(function () {
259 this.timeout(120000)
260
261 return runServers(strategy)
262 })
263
264 it('Should have 1 webseed on the first video', async function () {
265 await check1WebSeed(strategy)
266 await checkStatsWith1Webseed(strategy)
267 })
268
269 it('Should enable redundancy on server 1', function () {
270 return enableRedundancyOnServer1()
271 })
272
273 it('Should have 2 webseed on the first video', async function () {
274 this.timeout(40000)
275
276 await waitJobs(servers)
277 await waitUntilLog(servers[0], 'Duplicated ', 4)
278 await waitJobs(servers)
279
280 await check2Webseeds(strategy)
281 await checkStatsWith2Webseed(strategy)
282 })
283
284 it('Should unfollow on server 1 and remove duplicated videos', async function () {
285 this.timeout(40000)
286
287 await unfollow(servers[0].url, servers[0].accessToken, servers[1])
288
289 await waitJobs(servers)
290 await wait(5000)
291
292 await check1WebSeed(strategy)
293
294 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
295 })
296
297 after(function () {
298 return cleanServers()
299 })
300 })
301
302 describe('With recently added strategy', function () {
303 const strategy = 'recently-added'
304
305 before(function () {
306 this.timeout(120000)
307
308 return runServers(strategy, { min_views: 3 })
309 })
310
311 it('Should have 1 webseed on the first video', async function () {
312 await check1WebSeed(strategy)
313 await checkStatsWith1Webseed(strategy)
314 })
315
316 it('Should enable redundancy on server 1', function () {
317 return enableRedundancyOnServer1()
318 })
319
320 it('Should still have 1 webseed on the first video', async function () {
321 this.timeout(40000)
322
323 await waitJobs(servers)
324 await wait(15000)
325 await waitJobs(servers)
326
327 await check1WebSeed(strategy)
328 await checkStatsWith1Webseed(strategy)
329 })
330
331 it('Should view 2 times the first video to have > min_views config', async function () {
332 this.timeout(40000)
333
334 await viewVideo(servers[ 0 ].url, video1Server2UUID)
335 await viewVideo(servers[ 2 ].url, video1Server2UUID)
336
337 await wait(10000)
338 await waitJobs(servers)
339 })
340
341 it('Should have 2 webseed on the first video', async function () {
342 this.timeout(40000)
343
344 await waitJobs(servers)
345 await waitUntilLog(servers[0], 'Duplicated ', 4)
346 await waitJobs(servers)
347
348 await check2Webseeds(strategy)
349 await checkStatsWith2Webseed(strategy)
350 })
351
352 it('Should remove the video and the redundancy files', async function () {
353 this.timeout(20000)
354
355 await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
356
357 await waitJobs(servers)
358
359 for (const server of servers) {
360 await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber)
361 }
362 })
363
364 after(function () {
365 return cleanServers()
366 })
367 })
368
369 describe('Test expiration', function () {
370 const strategy = 'recently-added'
371
372 async function checkContains (servers: ServerInfo[], str: string) {
373 for (const server of servers) {
374 const res = await getVideo(server.url, video1Server2UUID)
375 const video: VideoDetails = res.body
376
377 for (const f of video.files) {
378 expect(f.magnetUri).to.contain(str)
379 }
380 }
381 }
382
383 async function checkNotContains (servers: ServerInfo[], str: string) {
384 for (const server of servers) {
385 const res = await getVideo(server.url, video1Server2UUID)
386 const video: VideoDetails = res.body
387
388 for (const f of video.files) {
389 expect(f.magnetUri).to.not.contain(str)
390 }
391 }
392 }
393
394 before(async function () {
395 this.timeout(120000)
396
397 await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
398
399 await enableRedundancyOnServer1()
400 })
401
402 it('Should still have 2 webseeds after 10 seconds', async function () {
403 this.timeout(40000)
404
405 await wait(10000)
406
407 try {
408 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
409 } catch {
410 // Maybe a server deleted a redundancy in the scheduler
411 await wait(2000)
412
413 await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
414 }
415 })
416
417 it('Should stop server 1 and expire video redundancy', async function () {
418 this.timeout(40000)
419
420 killallServers([ servers[0] ])
421
422 await wait(10000)
423
424 await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
425 })
426
427 after(function () {
428 return killallServers([ servers[1], servers[2] ])
429 })
430 })
431
432 describe('Test file replacement', function () {
433 let video2Server2UUID: string
434 const strategy = 'recently-added'
435
436 before(async function () {
437 this.timeout(120000)
438
439 await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
440
441 await enableRedundancyOnServer1()
442
443 await waitJobs(servers)
444 await waitUntilLog(servers[0], 'Duplicated ', 4)
445 await waitJobs(servers)
446
447 await check2Webseeds(strategy)
448 await checkStatsWith2Webseed(strategy)
449
450 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
451 video2Server2UUID = res.body.video.uuid
452 })
453
454 it('Should cache video 2 webseed on the first video', async function () {
455 this.timeout(50000)
456
457 await waitJobs(servers)
458
459 await wait(7000)
460
461 try {
462 await check1WebSeed(strategy, video1Server2UUID)
463 await check2Webseeds(strategy, video2Server2UUID)
464 } catch {
465 await wait(3000)
466
467 try {
468 await check1WebSeed(strategy, video1Server2UUID)
469 await check2Webseeds(strategy, video2Server2UUID)
470 } catch {
471 await wait(5000)
472
473 await check1WebSeed(strategy, video1Server2UUID)
474 await check2Webseeds(strategy, video2Server2UUID)
475 }
476 }
477 })
478
479 after(function () {
480 return cleanServers()
481 })
482 })
483})
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index c74c68a33..eeb8b7a28 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -3,7 +3,6 @@ import './email'
3import './follows' 3import './follows'
4import './handle-down' 4import './handle-down'
5import './jobs' 5import './jobs'
6import './redundancy'
7import './reverse-proxy' 6import './reverse-proxy'
8import './stats' 7import './stats'
9import './tracker' 8import './tracker'