]>
Commit | Line | Data |
---|---|---|
26370ce4 C |
1 | /* tslint:disable:no-unused-expression */ |
2 | ||
3 | import * as chai from 'chai' | |
4 | import 'mocha' | |
5 | import { VideoDetails } from '../../../../shared/models/videos' | |
6 | import { | |
7 | doubleFollow, | |
8 | flushAndRunMultipleServers, | |
9 | getFollowingListPaginationAndSort, | |
10 | getVideo, | |
11 | immutableAssign, | |
12 | killallServers, makeGetRequest, | |
13 | root, | |
14 | ServerInfo, | |
15 | setAccessTokensToServers, unfollow, | |
16 | uploadVideo, | |
17 | viewVideo, | |
18 | wait, | |
19 | waitUntilLog, | |
4c280004 | 20 | checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer, checkSegmentHash |
9639bd17 | 21 | } from '../../../../shared/utils' |
22 | import { waitJobs } from '../../../../shared/utils/server/jobs' | |
b9f23437 | 23 | |
26370ce4 | 24 | import * as magnetUtil from 'magnet-uri' |
9639bd17 | 25 | import { updateRedundancy } from '../../../../shared/utils/server/redundancy' |
26370ce4 C |
26 | import { ActorFollow } from '../../../../shared/models/actors' |
27 | import { readdir } from 'fs-extra' | |
28 | import { join } from 'path' | |
29 | import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' | |
9639bd17 | 30 | import { getStats } from '../../../../shared/utils/server/stats' |
26370ce4 C |
31 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' |
32 | ||
33 | const expect = chai.expect | |
34 | ||
35 | let servers: ServerInfo[] = [] | |
36 | let video1Server2UUID: string | |
37 | ||
38 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { | |
39 | const parsed = magnetUtil.decode(file.magnetUri) | |
40 | ||
41 | for (const ws of baseWebseeds) { | |
42 | const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) | |
43 | expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined | |
44 | } | |
45 | ||
46 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) | |
47 | } | |
48 | ||
49 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { | |
50 | const config = { | |
09209296 C |
51 | transcoding: { |
52 | hls: { | |
53 | enabled: true | |
54 | } | |
55 | }, | |
26370ce4 C |
56 | redundancy: { |
57 | videos: { | |
58 | check_interval: '5 seconds', | |
59 | strategies: [ | |
60 | immutableAssign({ | |
61 | min_lifetime: '1 hour', | |
62 | strategy: strategy, | |
6cb3482c | 63 | size: '200KB' |
26370ce4 C |
64 | }, additionalParams) |
65 | ] | |
66 | } | |
67 | } | |
68 | } | |
69 | servers = await flushAndRunMultipleServers(3, config) | |
70 | ||
71 | // Get the access tokens | |
72 | await setAccessTokensToServers(servers) | |
73 | ||
74 | { | |
75 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | |
76 | video1Server2UUID = res.body.video.uuid | |
77 | ||
78 | await viewVideo(servers[ 1 ].url, video1Server2UUID) | |
79 | } | |
80 | ||
81 | await waitJobs(servers) | |
82 | ||
83 | // Server 1 and server 2 follow each other | |
84 | await doubleFollow(servers[ 0 ], servers[ 1 ]) | |
85 | // Server 1 and server 3 follow each other | |
86 | await doubleFollow(servers[ 0 ], servers[ 2 ]) | |
87 | // Server 2 and server 3 follow each other | |
88 | await doubleFollow(servers[ 1 ], servers[ 2 ]) | |
89 | ||
90 | await waitJobs(servers) | |
91 | } | |
92 | ||
09209296 | 93 | async function check1WebSeed (videoUUID?: string) { |
26370ce4 C |
94 | if (!videoUUID) videoUUID = video1Server2UUID |
95 | ||
96 | const webseeds = [ | |
97 | 'http://localhost:9002/static/webseed/' + videoUUID | |
98 | ] | |
99 | ||
100 | for (const server of servers) { | |
09209296 C |
101 | // With token to avoid issues with video follow constraints |
102 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) | |
26370ce4 | 103 | |
09209296 C |
104 | const video: VideoDetails = res.body |
105 | for (const f of video.files) { | |
106 | checkMagnetWebseeds(f, webseeds, server) | |
26370ce4 C |
107 | } |
108 | } | |
109 | } | |
110 | ||
09209296 | 111 | async function check2Webseeds (videoUUID?: string) { |
26370ce4 C |
112 | if (!videoUUID) videoUUID = video1Server2UUID |
113 | ||
114 | const webseeds = [ | |
b9fffa29 | 115 | 'http://localhost:9001/static/redundancy/' + videoUUID, |
26370ce4 C |
116 | 'http://localhost:9002/static/webseed/' + videoUUID |
117 | ] | |
118 | ||
119 | for (const server of servers) { | |
120 | const res = await getVideo(server.url, videoUUID) | |
121 | ||
122 | const video: VideoDetails = res.body | |
123 | ||
124 | for (const file of video.files) { | |
125 | checkMagnetWebseeds(file, webseeds, server) | |
126 | ||
b9fffa29 C |
127 | await makeGetRequest({ |
128 | url: servers[0].url, | |
129 | statusCodeExpected: 200, | |
130 | path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`, | |
131 | contentType: null | |
132 | }) | |
133 | await makeGetRequest({ | |
134 | url: servers[1].url, | |
135 | statusCodeExpected: 200, | |
09209296 | 136 | path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, |
b9fffa29 C |
137 | contentType: null |
138 | }) | |
26370ce4 C |
139 | } |
140 | } | |
141 | ||
b9fffa29 C |
142 | for (const directory of [ 'test1/redundancy', 'test2/videos' ]) { |
143 | const files = await readdir(join(root(), directory)) | |
26370ce4 C |
144 | expect(files).to.have.length.at.least(4) |
145 | ||
146 | for (const resolution of [ 240, 360, 480, 720 ]) { | |
147 | expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined | |
148 | } | |
149 | } | |
150 | } | |
151 | ||
09209296 C |
152 | async function check0PlaylistRedundancies (videoUUID?: string) { |
153 | if (!videoUUID) videoUUID = video1Server2UUID | |
154 | ||
155 | for (const server of servers) { | |
156 | // With token to avoid issues with video follow constraints | |
157 | const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) | |
158 | const video: VideoDetails = res.body | |
159 | ||
160 | expect(video.streamingPlaylists).to.be.an('array') | |
161 | expect(video.streamingPlaylists).to.have.lengthOf(1) | |
162 | expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) | |
163 | } | |
164 | } | |
165 | ||
166 | async function check1PlaylistRedundancies (videoUUID?: string) { | |
167 | if (!videoUUID) videoUUID = video1Server2UUID | |
168 | ||
169 | for (const server of servers) { | |
170 | const res = await getVideo(server.url, videoUUID) | |
171 | const video: VideoDetails = res.body | |
172 | ||
173 | expect(video.streamingPlaylists).to.have.lengthOf(1) | |
174 | expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) | |
175 | ||
176 | const redundancy = video.streamingPlaylists[0].redundancies[0] | |
177 | ||
178 | expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) | |
179 | } | |
180 | ||
4c280004 C |
181 | const baseUrlPlaylist = servers[1].url + '/static/playlists/hls' |
182 | const baseUrlSegment = servers[0].url + '/static/redundancy/hls' | |
183 | ||
184 | const res = await getVideo(servers[0].url, videoUUID) | |
185 | const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0] | |
186 | ||
187 | for (const resolution of [ 240, 360, 480, 720 ]) { | |
188 | await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist) | |
189 | } | |
09209296 C |
190 | |
191 | for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { | |
192 | const files = await readdir(join(root(), directory, videoUUID)) | |
193 | expect(files).to.have.length.at.least(4) | |
194 | ||
195 | for (const resolution of [ 240, 360, 480, 720 ]) { | |
4c280004 C |
196 | const filename = `${videoUUID}-${resolution}-fragmented.mp4` |
197 | ||
198 | expect(files.find(f => f === filename)).to.not.be.undefined | |
09209296 C |
199 | } |
200 | } | |
201 | } | |
202 | ||
203 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | |
204 | const res = await getStats(servers[0].url) | |
205 | const data: ServerStats = res.body | |
206 | ||
207 | expect(data.videosRedundancy).to.have.lengthOf(1) | |
208 | const stat = data.videosRedundancy[0] | |
209 | ||
210 | expect(stat.strategy).to.equal(strategy) | |
211 | expect(stat.totalSize).to.equal(204800) | |
212 | expect(stat.totalUsed).to.be.at.least(1).and.below(204801) | |
213 | expect(stat.totalVideoFiles).to.equal(4) | |
214 | expect(stat.totalVideos).to.equal(1) | |
215 | } | |
216 | ||
217 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | |
218 | const res = await getStats(servers[0].url) | |
219 | const data: ServerStats = res.body | |
220 | ||
221 | expect(data.videosRedundancy).to.have.lengthOf(1) | |
222 | ||
223 | const stat = data.videosRedundancy[0] | |
224 | expect(stat.strategy).to.equal(strategy) | |
225 | expect(stat.totalSize).to.equal(204800) | |
226 | expect(stat.totalUsed).to.equal(0) | |
227 | expect(stat.totalVideoFiles).to.equal(0) | |
228 | expect(stat.totalVideos).to.equal(0) | |
229 | } | |
230 | ||
26370ce4 C |
231 | async function enableRedundancyOnServer1 () { |
232 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | |
233 | ||
234 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') | |
235 | const follows: ActorFollow[] = res.body.data | |
236 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | |
237 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | |
238 | ||
239 | expect(server3).to.not.be.undefined | |
240 | expect(server3.following.hostRedundancyAllowed).to.be.false | |
241 | ||
242 | expect(server2).to.not.be.undefined | |
243 | expect(server2.following.hostRedundancyAllowed).to.be.true | |
244 | } | |
245 | ||
246 | async function disableRedundancyOnServer1 () { | |
247 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) | |
248 | ||
249 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') | |
250 | const follows: ActorFollow[] = res.body.data | |
251 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | |
252 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | |
253 | ||
254 | expect(server3).to.not.be.undefined | |
255 | expect(server3.following.hostRedundancyAllowed).to.be.false | |
256 | ||
257 | expect(server2).to.not.be.undefined | |
258 | expect(server2.following.hostRedundancyAllowed).to.be.false | |
259 | } | |
260 | ||
261 | async function cleanServers () { | |
262 | killallServers(servers) | |
263 | } | |
264 | ||
265 | describe('Test videos redundancy', function () { | |
266 | ||
267 | describe('With most-views strategy', function () { | |
268 | const strategy = 'most-views' | |
269 | ||
270 | before(function () { | |
271 | this.timeout(120000) | |
272 | ||
273 | return runServers(strategy) | |
274 | }) | |
275 | ||
276 | it('Should have 1 webseed on the first video', async function () { | |
09209296 C |
277 | await check1WebSeed() |
278 | await check0PlaylistRedundancies() | |
26370ce4 C |
279 | await checkStatsWith1Webseed(strategy) |
280 | }) | |
281 | ||
282 | it('Should enable redundancy on server 1', function () { | |
283 | return enableRedundancyOnServer1() | |
284 | }) | |
285 | ||
6cb3482c | 286 | it('Should have 2 webseeds on the first video', async function () { |
09209296 | 287 | this.timeout(80000) |
26370ce4 C |
288 | |
289 | await waitJobs(servers) | |
09209296 | 290 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
26370ce4 C |
291 | await waitJobs(servers) |
292 | ||
09209296 C |
293 | await check2Webseeds() |
294 | await check1PlaylistRedundancies() | |
26370ce4 C |
295 | await checkStatsWith2Webseed(strategy) |
296 | }) | |
297 | ||
298 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { | |
09209296 | 299 | this.timeout(80000) |
26370ce4 C |
300 | |
301 | await disableRedundancyOnServer1() | |
302 | ||
303 | await waitJobs(servers) | |
304 | await wait(5000) | |
305 | ||
09209296 C |
306 | await check1WebSeed() |
307 | await check0PlaylistRedundancies() | |
26370ce4 | 308 | |
09209296 | 309 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ]) |
26370ce4 C |
310 | }) |
311 | ||
312 | after(function () { | |
313 | return cleanServers() | |
314 | }) | |
315 | }) | |
316 | ||
317 | describe('With trending strategy', function () { | |
318 | const strategy = 'trending' | |
319 | ||
320 | before(function () { | |
321 | this.timeout(120000) | |
322 | ||
323 | return runServers(strategy) | |
324 | }) | |
325 | ||
326 | it('Should have 1 webseed on the first video', async function () { | |
09209296 C |
327 | await check1WebSeed() |
328 | await check0PlaylistRedundancies() | |
26370ce4 C |
329 | await checkStatsWith1Webseed(strategy) |
330 | }) | |
331 | ||
332 | it('Should enable redundancy on server 1', function () { | |
333 | return enableRedundancyOnServer1() | |
334 | }) | |
335 | ||
6cb3482c | 336 | it('Should have 2 webseeds on the first video', async function () { |
09209296 | 337 | this.timeout(80000) |
26370ce4 C |
338 | |
339 | await waitJobs(servers) | |
09209296 | 340 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
26370ce4 C |
341 | await waitJobs(servers) |
342 | ||
09209296 C |
343 | await check2Webseeds() |
344 | await check1PlaylistRedundancies() | |
26370ce4 C |
345 | await checkStatsWith2Webseed(strategy) |
346 | }) | |
347 | ||
348 | it('Should unfollow on server 1 and remove duplicated videos', async function () { | |
09209296 | 349 | this.timeout(80000) |
26370ce4 C |
350 | |
351 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) | |
352 | ||
353 | await waitJobs(servers) | |
354 | await wait(5000) | |
355 | ||
09209296 C |
356 | await check1WebSeed() |
357 | await check0PlaylistRedundancies() | |
26370ce4 C |
358 | |
359 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | |
360 | }) | |
361 | ||
362 | after(function () { | |
363 | return cleanServers() | |
364 | }) | |
365 | }) | |
366 | ||
367 | describe('With recently added strategy', function () { | |
368 | const strategy = 'recently-added' | |
369 | ||
370 | before(function () { | |
371 | this.timeout(120000) | |
372 | ||
373 | return runServers(strategy, { min_views: 3 }) | |
374 | }) | |
375 | ||
376 | it('Should have 1 webseed on the first video', async function () { | |
09209296 C |
377 | await check1WebSeed() |
378 | await check0PlaylistRedundancies() | |
26370ce4 C |
379 | await checkStatsWith1Webseed(strategy) |
380 | }) | |
381 | ||
382 | it('Should enable redundancy on server 1', function () { | |
383 | return enableRedundancyOnServer1() | |
384 | }) | |
385 | ||
386 | it('Should still have 1 webseed on the first video', async function () { | |
09209296 | 387 | this.timeout(80000) |
26370ce4 C |
388 | |
389 | await waitJobs(servers) | |
390 | await wait(15000) | |
391 | await waitJobs(servers) | |
392 | ||
09209296 C |
393 | await check1WebSeed() |
394 | await check0PlaylistRedundancies() | |
26370ce4 C |
395 | await checkStatsWith1Webseed(strategy) |
396 | }) | |
397 | ||
398 | it('Should view 2 times the first video to have > min_views config', async function () { | |
09209296 | 399 | this.timeout(80000) |
26370ce4 C |
400 | |
401 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | |
402 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | |
403 | ||
404 | await wait(10000) | |
405 | await waitJobs(servers) | |
406 | }) | |
407 | ||
6cb3482c | 408 | it('Should have 2 webseeds on the first video', async function () { |
09209296 | 409 | this.timeout(80000) |
26370ce4 C |
410 | |
411 | await waitJobs(servers) | |
09209296 | 412 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
26370ce4 C |
413 | await waitJobs(servers) |
414 | ||
09209296 C |
415 | await check2Webseeds() |
416 | await check1PlaylistRedundancies() | |
26370ce4 C |
417 | await checkStatsWith2Webseed(strategy) |
418 | }) | |
419 | ||
420 | it('Should remove the video and the redundancy files', async function () { | |
421 | this.timeout(20000) | |
422 | ||
423 | await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID) | |
424 | ||
425 | await waitJobs(servers) | |
426 | ||
427 | for (const server of servers) { | |
428 | await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber) | |
429 | } | |
430 | }) | |
431 | ||
432 | after(function () { | |
433 | return cleanServers() | |
434 | }) | |
435 | }) | |
436 | ||
437 | describe('Test expiration', function () { | |
438 | const strategy = 'recently-added' | |
439 | ||
440 | async function checkContains (servers: ServerInfo[], str: string) { | |
441 | for (const server of servers) { | |
442 | const res = await getVideo(server.url, video1Server2UUID) | |
443 | const video: VideoDetails = res.body | |
444 | ||
445 | for (const f of video.files) { | |
446 | expect(f.magnetUri).to.contain(str) | |
447 | } | |
448 | } | |
449 | } | |
450 | ||
451 | async function checkNotContains (servers: ServerInfo[], str: string) { | |
452 | for (const server of servers) { | |
453 | const res = await getVideo(server.url, video1Server2UUID) | |
454 | const video: VideoDetails = res.body | |
455 | ||
456 | for (const f of video.files) { | |
457 | expect(f.magnetUri).to.not.contain(str) | |
458 | } | |
459 | } | |
460 | } | |
461 | ||
462 | before(async function () { | |
463 | this.timeout(120000) | |
464 | ||
465 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | |
466 | ||
467 | await enableRedundancyOnServer1() | |
468 | }) | |
469 | ||
470 | it('Should still have 2 webseeds after 10 seconds', async function () { | |
09209296 | 471 | this.timeout(80000) |
26370ce4 C |
472 | |
473 | await wait(10000) | |
474 | ||
475 | try { | |
476 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | |
477 | } catch { | |
478 | // Maybe a server deleted a redundancy in the scheduler | |
479 | await wait(2000) | |
480 | ||
481 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | |
482 | } | |
483 | }) | |
484 | ||
485 | it('Should stop server 1 and expire video redundancy', async function () { | |
09209296 | 486 | this.timeout(80000) |
26370ce4 C |
487 | |
488 | killallServers([ servers[0] ]) | |
489 | ||
6cb3482c | 490 | await wait(15000) |
26370ce4 C |
491 | |
492 | await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') | |
493 | }) | |
494 | ||
495 | after(function () { | |
496 | return killallServers([ servers[1], servers[2] ]) | |
497 | }) | |
498 | }) | |
499 | ||
500 | describe('Test file replacement', function () { | |
501 | let video2Server2UUID: string | |
502 | const strategy = 'recently-added' | |
503 | ||
504 | before(async function () { | |
505 | this.timeout(120000) | |
506 | ||
507 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | |
508 | ||
509 | await enableRedundancyOnServer1() | |
510 | ||
511 | await waitJobs(servers) | |
09209296 | 512 | await waitUntilLog(servers[0], 'Duplicated ', 5) |
26370ce4 C |
513 | await waitJobs(servers) |
514 | ||
09209296 C |
515 | await check2Webseeds() |
516 | await check1PlaylistRedundancies() | |
26370ce4 C |
517 | await checkStatsWith2Webseed(strategy) |
518 | ||
519 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | |
520 | video2Server2UUID = res.body.video.uuid | |
521 | }) | |
522 | ||
6cb3482c C |
523 | it('Should cache video 2 webseeds on the first video', async function () { |
524 | this.timeout(120000) | |
26370ce4 C |
525 | |
526 | await waitJobs(servers) | |
527 | ||
6cb3482c | 528 | let checked = false |
26370ce4 | 529 | |
6cb3482c C |
530 | while (checked === false) { |
531 | await wait(1000) | |
26370ce4 C |
532 | |
533 | try { | |
09209296 C |
534 | await check1WebSeed(video1Server2UUID) |
535 | await check0PlaylistRedundancies(video1Server2UUID) | |
536 | await check2Webseeds(video2Server2UUID) | |
537 | await check1PlaylistRedundancies(video2Server2UUID) | |
26370ce4 | 538 | |
6cb3482c C |
539 | checked = true |
540 | } catch { | |
541 | checked = false | |
26370ce4 C |
542 | } |
543 | } | |
544 | }) | |
545 | ||
09209296 C |
546 | it('Should disable strategy and remove redundancies', async function () { |
547 | this.timeout(80000) | |
548 | ||
549 | await waitJobs(servers) | |
550 | ||
551 | killallServers([ servers[ 0 ] ]) | |
552 | await reRunServer(servers[ 0 ], { | |
553 | redundancy: { | |
554 | videos: { | |
555 | check_interval: '1 second', | |
556 | strategies: [] | |
557 | } | |
558 | } | |
559 | }) | |
560 | ||
561 | await waitJobs(servers) | |
562 | ||
563 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ]) | |
564 | }) | |
565 | ||
26370ce4 C |
566 | after(function () { |
567 | return cleanServers() | |
568 | }) | |
569 | }) | |
570 | }) |