server/tests/api/live/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { FfmpegCommand } from 'fluent-ffmpeg'
   6 import { join } from 'path'
   7 import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
   8 import { getLiveNotificationSocket } from '@shared/extra-utils/socket/socket-io'
   9 import { LiveVideo, LiveVideoCreate, Video, VideoDetails, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models'
  10 import {
  11   addVideoToBlacklist,
  12   buildServerDirectory,
  13   checkLiveCleanup,
  14   checkLiveSegmentHash,
  15   checkResolutionsInMasterPlaylist,
  16   cleanupTests,
  17   createLive,
  18   doubleFollow,
  19   flushAndRunMultipleServers,
  20   getLive,
  21   getPlaylist,
  22   getVideo,
  23   getVideoIdFromUUID,
  24   getVideosList,
  25   killallServers,
  26   makeRawRequest,
  27   removeVideo,
  28   reRunServer,
  29   sendRTMPStream,
  30   sendRTMPStreamInVideo,
  31   ServerInfo,
  32   setAccessTokensToServers,
  33   setDefaultVideoChannel,
  34   stopFfmpeg,
  35   testFfmpegStreamError,
  36   testImage,
  37   updateCustomSubConfig,
  38   updateLive,
  39   viewVideo,
  40   wait,
  41   waitJobs,
  42   waitUntilLiveStarts,
  43   waitUntilLog
  44 } from '../../../../shared/extra-utils'
  45
  46 const expect = chai.expect
  47
  48 describe('Test live', function () {
  49   let servers: ServerInfo[] = []
  50
  51   before(async function () {
  52     this.timeout(120000)
  53
  54     servers = await flushAndRunMultipleServers(2)
  55
  56     // Get the access tokens
  57     await setAccessTokensToServers(servers)
  58     await setDefaultVideoChannel(servers)
  59
  60     await updateCustomSubConfig(servers[0].url, servers[0].accessToken, {
  61       live: {
  62         enabled: true,
  63         allowReplay: true,
  64         transcoding: {
  65           enabled: false
  66         }
  67       }
  68     })
  69
  70     // Server 1 and server 2 follow each other
  71     await doubleFollow(servers[0], servers[1])
  72   })
  73
  74   describe('Live creation, update and delete', function () {
  75     let liveVideoUUID: string
  76
  77     it('Should create a live with the appropriate parameters', async function () {
  78       this.timeout(20000)
  79
  80       const attributes: LiveVideoCreate = {
  81         category: 1,
  82         licence: 2,
  83         language: 'fr',
  84         description: 'super live description',
  85         support: 'support field',
  86         channelId: servers[0].videoChannel.id,
  87         nsfw: false,
  88         waitTranscoding: false,
  89         name: 'my super live',
  90         tags: [ 'tag1', 'tag2' ],
  91         commentsEnabled: false,
  92         downloadEnabled: false,
  93         saveReplay: true,
  94         privacy: VideoPrivacy.PUBLIC,
  95         previewfile: 'video_short1-preview.webm.jpg',
  96         thumbnailfile: 'video_short1.webm.jpg'
  97       }
  98
  99       const res = await createLive(servers[0].url, servers[0].accessToken, attributes)
 100       liveVideoUUID = res.body.video.uuid
 101
 102       await waitJobs(servers)
 103
 104       for (const server of servers) {
 105         const resVideo = await getVideo(server.url, liveVideoUUID)
 106         const video: VideoDetails = resVideo.body
 107
 108         expect(video.category.id).to.equal(1)
 109         expect(video.licence.id).to.equal(2)
 110         expect(video.language.id).to.equal('fr')
 111         expect(video.description).to.equal('super live description')
 112         expect(video.support).to.equal('support field')
 113
 114         expect(video.channel.name).to.equal(servers[0].videoChannel.name)
 115         expect(video.channel.host).to.equal(servers[0].videoChannel.host)
 116
 117         expect(video.nsfw).to.be.false
 118         expect(video.waitTranscoding).to.be.false
 119         expect(video.name).to.equal('my super live')
 120         expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
 121         expect(video.commentsEnabled).to.be.false
 122         expect(video.downloadEnabled).to.be.false
 123         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 124
 125         await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
 126         await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
 127
 128         const resLive = await getLive(server.url, server.accessToken, liveVideoUUID)
 129         const live: LiveVideo = resLive.body
 130
 131         if (server.url === servers[0].url) {
 132           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 133           expect(live.streamKey).to.not.be.empty
 134         } else {
 135           expect(live.rtmpUrl).to.be.null
 136           expect(live.streamKey).to.be.null
 137         }
 138
 139         expect(live.saveReplay).to.be.true
 140       }
 141     })
 142
 143     it('Should have a default preview and thumbnail', async function () {
 144       this.timeout(20000)
 145
 146       const attributes: LiveVideoCreate = {
 147         name: 'default live thumbnail',
 148         channelId: servers[0].videoChannel.id,
 149         privacy: VideoPrivacy.UNLISTED,
 150         nsfw: true
 151       }
 152
 153       const res = await createLive(servers[0].url, servers[0].accessToken, attributes)
 154       const videoId = res.body.video.uuid
 155
 156       await waitJobs(servers)
 157
 158       for (const server of servers) {
 159         const resVideo = await getVideo(server.url, videoId)
 160         const video: VideoDetails = resVideo.body
 161
 162         expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
 163         expect(video.nsfw).to.be.true
 164
 165         await makeRawRequest(server.url + video.thumbnailPath, 200)
 166         await makeRawRequest(server.url + video.previewPath, 200)
 167       }
 168     })
 169
 170     it('Should not have the live listed since nobody streams into', async function () {
 171       for (const server of servers) {
 172         const res = await getVideosList(server.url)
 173
 174         expect(res.body.total).to.equal(0)
 175         expect(res.body.data).to.have.lengthOf(0)
 176       }
 177     })
 178
 179     it('Should not be able to update a live of another server', async function () {
 180       await updateLive(servers[1].url, servers[1].accessToken, liveVideoUUID, { saveReplay: false }, 403)
 181     })
 182
 183     it('Should update the live', async function () {
 184       this.timeout(10000)
 185
 186       await updateLive(servers[0].url, servers[0].accessToken, liveVideoUUID, { saveReplay: false })
 187       await waitJobs(servers)
 188     })
 189
 190     it('Have the live updated', async function () {
 191       for (const server of servers) {
 192         const res = await getLive(server.url, server.accessToken, liveVideoUUID)
 193         const live: LiveVideo = res.body
 194
 195         if (server.url === servers[0].url) {
 196           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 197           expect(live.streamKey).to.not.be.empty
 198         } else {
 199           expect(live.rtmpUrl).to.be.null
 200           expect(live.streamKey).to.be.null
 201         }
 202
 203         expect(live.saveReplay).to.be.false
 204       }
 205     })
 206
 207     it('Delete the live', async function () {
 208       this.timeout(10000)
 209
 210       await removeVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 211       await waitJobs(servers)
 212     })
 213
 214     it('Should have the live deleted', async function () {
 215       for (const server of servers) {
 216         await getVideo(server.url, liveVideoUUID, 404)
 217         await getLive(server.url, server.accessToken, liveVideoUUID, 404)
 218       }
 219     })
 220   })
 221
 222   describe('Stream checks', function () {
 223     let liveVideo: LiveVideo & VideoDetails
 224     let rtmpUrl: string
 225
 226     before(function () {
 227       rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + ''
 228     })
 229
 230     async function createLiveWrapper () {
 231       const liveAttributes = {
 232         name: 'user live',
 233         channelId: servers[0].videoChannel.id,
 234         privacy: VideoPrivacy.PUBLIC,
 235         saveReplay: false
 236       }
 237
 238       const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
 239       const uuid = res.body.video.uuid
 240
 241       const resLive = await getLive(servers[0].url, servers[0].accessToken, uuid)
 242       const resVideo = await getVideo(servers[0].url, uuid)
 243
 244       return Object.assign(resVideo.body, resLive.body) as LiveVideo & VideoDetails
 245     }
 246
 247     it('Should not allow a stream without the appropriate path', async function () {
 248       this.timeout(30000)
 249
 250       liveVideo = await createLiveWrapper()
 251
 252       const command = sendRTMPStream(rtmpUrl + '/bad-live', liveVideo.streamKey)
 253       await testFfmpegStreamError(command, true)
 254     })
 255
 256     it('Should not allow a stream without the appropriate stream key', async function () {
 257       this.timeout(30000)
 258
 259       const command = sendRTMPStream(rtmpUrl + '/live', 'bad-stream-key')
 260       await testFfmpegStreamError(command, true)
 261     })
 262
 263     it('Should succeed with the correct params', async function () {
 264       this.timeout(30000)
 265
 266       const command = sendRTMPStream(rtmpUrl + '/live', liveVideo.streamKey)
 267       await testFfmpegStreamError(command, false)
 268     })
 269
 270     it('Should not allow a stream on a live that was blacklisted', async function () {
 271       this.timeout(30000)
 272
 273       liveVideo = await createLiveWrapper()
 274
 275       await addVideoToBlacklist(servers[0].url, servers[0].accessToken, liveVideo.uuid)
 276
 277       const command = sendRTMPStream(rtmpUrl + '/live', liveVideo.streamKey)
 278       await testFfmpegStreamError(command, true)
 279     })
 280
 281     it('Should not allow a stream on a live that was deleted', async function () {
 282       this.timeout(30000)
 283
 284       liveVideo = await createLiveWrapper()
 285
 286       await removeVideo(servers[0].url, servers[0].accessToken, liveVideo.uuid)
 287
 288       const command = sendRTMPStream(rtmpUrl + '/live', liveVideo.streamKey)
 289       await testFfmpegStreamError(command, true)
 290     })
 291   })
 292
 293   describe('Live transcoding', function () {
 294     let liveVideoId: string
 295
 296     async function createLiveWrapper (saveReplay: boolean) {
 297       const liveAttributes = {
 298         name: 'live video',
 299         channelId: servers[0].videoChannel.id,
 300         privacy: VideoPrivacy.PUBLIC,
 301         saveReplay
 302       }
 303
 304       const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
 305       return res.body.video.uuid
 306     }
 307
 308     async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
 309       for (const server of servers) {
 310         const resList = await getVideosList(server.url)
 311         const videos: Video[] = resList.body.data
 312
 313         expect(videos.find(v => v.uuid === liveVideoId)).to.exist
 314
 315         const resVideo = await getVideo(server.url, liveVideoId)
 316         const video: VideoDetails = resVideo.body
 317
 318         expect(video.streamingPlaylists).to.have.lengthOf(1)
 319
 320         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 321         expect(hlsPlaylist).to.exist
 322
 323         // Only finite files are displayed
 324         expect(hlsPlaylist.files).to.have.lengthOf(0)
 325
 326         await checkResolutionsInMasterPlaylist(hlsPlaylist.playlistUrl, resolutions)
 327
 328         for (let i = 0; i < resolutions.length; i++) {
 329           const segmentName = `${i}-000001.ts`
 330           await waitUntilLog(servers[0], `${video.uuid}/${segmentName}`, 2, false)
 331
 332           const res = await getPlaylist(`${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`)
 333           const subPlaylist = res.text
 334
 335           expect(subPlaylist).to.contain(segmentName)
 336
 337           const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
 338           await checkLiveSegmentHash(baseUrlAndPath, video.uuid, segmentName, hlsPlaylist)
 339         }
 340       }
 341     }
 342
 343     function updateConf (resolutions: number[]) {
 344       return updateCustomSubConfig(servers[0].url, servers[0].accessToken, {
 345         live: {
 346           enabled: true,
 347           allowReplay: true,
 348           maxDuration: null,
 349           transcoding: {
 350             enabled: true,
 351             resolutions: {
 352               '240p': resolutions.includes(240),
 353               '360p': resolutions.includes(360),
 354               '480p': resolutions.includes(480),
 355               '720p': resolutions.includes(720),
 356               '1080p': resolutions.includes(1080),
 357               '2160p': resolutions.includes(2160)
 358             }
 359           }
 360         }
 361       })
 362     }
 363
 364     before(async function () {
 365       await updateConf([])
 366     })
 367
 368     it('Should enable transcoding without additional resolutions', async function () {
 369       this.timeout(30000)
 370
 371       liveVideoId = await createLiveWrapper(false)
 372
 373       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
 374       await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId)
 375       await waitJobs(servers)
 376
 377       await testVideoResolutions(liveVideoId, [ 720 ])
 378
 379       await stopFfmpeg(command)
 380     })
 381
 382     it('Should enable transcoding with some resolutions', async function () {
 383       this.timeout(30000)
 384
 385       const resolutions = [ 240, 480 ]
 386       await updateConf(resolutions)
 387       liveVideoId = await createLiveWrapper(false)
 388
 389       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
 390       await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId)
 391       await waitJobs(servers)
 392
 393       await testVideoResolutions(liveVideoId, resolutions)
 394
 395       await stopFfmpeg(command)
 396     })
 397
 398     it('Should enable transcoding with some resolutions and correctly save them', async function () {
 399       this.timeout(60000)
 400
 401       const resolutions = [ 240, 360, 720 ]
 402
 403       await updateConf(resolutions)
 404       liveVideoId = await createLiveWrapper(true)
 405
 406       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm')
 407       await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId)
 408       await waitJobs(servers)
 409
 410       await testVideoResolutions(liveVideoId, resolutions)
 411
 412       await stopFfmpeg(command)
 413
 414       await waitJobs(servers)
 415
 416       const bitrateLimits = {
 417         720: 2800 * 1000,
 418         360: 780 * 1000,
 419         240: 320 * 1000
 420       }
 421
 422       for (const server of servers) {
 423         const resVideo = await getVideo(server.url, liveVideoId)
 424         const video: VideoDetails = resVideo.body
 425
 426         expect(video.duration).to.be.greaterThan(1)
 427         expect(video.files).to.have.lengthOf(0)
 428
 429         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 430
 431         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 432
 433         for (const resolution of resolutions) {
 434           const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
 435
 436           expect(file).to.exist
 437           expect(file.fps).to.be.approximately(30, 5)
 438           expect(file.size).to.be.greaterThan(1)
 439
 440           const filename = `${video.uuid}-${resolution}-fragmented.mp4`
 441           const segmentPath = buildServerDirectory(servers[0], join('streaming-playlists', 'hls', video.uuid, filename))
 442
 443           const probe = await ffprobePromise(segmentPath)
 444           const videoStream = await getVideoStreamFromFile(segmentPath, probe)
 445           console.log(videoStream)
 446           expect(probe.format.bit_rate).to.be.below(bitrateLimits[videoStream.height])
 447
 448           await makeRawRequest(file.torrentUrl, 200)
 449           await makeRawRequest(file.fileUrl, 200)
 450         }
 451       }
 452     })
 453
 454     it('Should correctly have cleaned up the live files', async function () {
 455       this.timeout(30000)
 456
 457       await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ])
 458     })
 459   })
 460
 461   describe('Live views', function () {
 462     let liveVideoId: string
 463     let command: FfmpegCommand
 464
 465     async function countViews (expected: number) {
 466       for (const server of servers) {
 467         const res = await getVideo(server.url, liveVideoId)
 468         const video: VideoDetails = res.body
 469
 470         expect(video.views).to.equal(expected)
 471       }
 472     }
 473
 474     before(async function () {
 475       this.timeout(30000)
 476
 477       const liveAttributes = {
 478         name: 'live video',
 479         channelId: servers[0].videoChannel.id,
 480         privacy: VideoPrivacy.PUBLIC
 481       }
 482
 483       const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
 484       liveVideoId = res.body.video.uuid
 485
 486       command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
 487       await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId)
 488       await waitJobs(servers)
 489     })
 490
 491     it('Should display no views for a live', async function () {
 492       await countViews(0)
 493     })
 494
 495     it('Should view a live twice and display 1 view', async function () {
 496       this.timeout(30000)
 497
 498       await viewVideo(servers[0].url, liveVideoId)
 499       await viewVideo(servers[0].url, liveVideoId)
 500
 501       await wait(7000)
 502
 503       await waitJobs(servers)
 504
 505       await countViews(1)
 506     })
 507
 508     it('Should wait and display 0 views', async function () {
 509       this.timeout(30000)
 510
 511       await wait(7000)
 512       await waitJobs(servers)
 513
 514       await countViews(0)
 515     })
 516
 517     it('Should view a live on a remote and on local and display 2 views', async function () {
 518       this.timeout(30000)
 519
 520       await viewVideo(servers[0].url, liveVideoId)
 521       await viewVideo(servers[1].url, liveVideoId)
 522       await viewVideo(servers[1].url, liveVideoId)
 523
 524       await wait(7000)
 525       await waitJobs(servers)
 526
 527       await countViews(2)
 528     })
 529
 530     after(async function () {
 531       await stopFfmpeg(command)
 532     })
 533   })
 534
 535   describe('Live socket messages', function () {
 536
 537     async function createLiveWrapper () {
 538       const liveAttributes = {
 539         name: 'live video',
 540         channelId: servers[0].videoChannel.id,
 541         privacy: VideoPrivacy.PUBLIC
 542       }
 543
 544       const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
 545       return res.body.video.uuid
 546     }
 547
 548     it('Should correctly send a message when the live starts and ends', async function () {
 549       this.timeout(60000)
 550
 551       const localStateChanges: VideoState[] = []
 552       const remoteStateChanges: VideoState[] = []
 553
 554       const liveVideoUUID = await createLiveWrapper()
 555       await waitJobs(servers)
 556
 557       {
 558         const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
 559
 560         const localSocket = getLiveNotificationSocket(servers[0].url)
 561         localSocket.on('state-change', data => localStateChanges.push(data.state))
 562         localSocket.emit('subscribe', { videoId })
 563       }
 564
 565       {
 566         const videoId = await getVideoIdFromUUID(servers[1].url, liveVideoUUID)
 567
 568         const remoteSocket = getLiveNotificationSocket(servers[1].url)
 569         remoteSocket.on('state-change', data => remoteStateChanges.push(data.state))
 570         remoteSocket.emit('subscribe', { videoId })
 571       }
 572
 573       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 574       await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoUUID)
 575       await waitJobs(servers)
 576
 577       for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
 578         expect(stateChanges).to.have.lengthOf(1)
 579         expect(stateChanges[0]).to.equal(VideoState.PUBLISHED)
 580       }
 581
 582       await stopFfmpeg(command)
 583       await waitJobs(servers)
 584
 585       for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
 586         expect(stateChanges).to.have.lengthOf(2)
 587         expect(stateChanges[1]).to.equal(VideoState.LIVE_ENDED)
 588       }
 589     })
 590
 591     it('Should not receive a notification after unsubscribe', async function () {
 592       this.timeout(60000)
 593
 594       const stateChanges: VideoState[] = []
 595
 596       const liveVideoUUID = await createLiveWrapper()
 597       await waitJobs(servers)
 598
 599       const videoId = await getVideoIdFromUUID(servers[0].url, liveVideoUUID)
 600
 601       const socket = getLiveNotificationSocket(servers[0].url)
 602       socket.on('state-change', data => stateChanges.push(data.state))
 603       socket.emit('subscribe', { videoId })
 604
 605       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 606       await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoUUID)
 607       await waitJobs(servers)
 608
 609       expect(stateChanges).to.have.lengthOf(1)
 610       socket.emit('unsubscribe', { videoId })
 611
 612       await stopFfmpeg(command)
 613       await waitJobs(servers)
 614
 615       expect(stateChanges).to.have.lengthOf(1)
 616     })
 617   })
 618
 619   describe('After a server restart', function () {
 620     let liveVideoId: string
 621     let liveVideoReplayId: string
 622
 623     async function createLiveWrapper (saveReplay: boolean) {
 624       const liveAttributes = {
 625         name: 'live video',
 626         channelId: servers[0].videoChannel.id,
 627         privacy: VideoPrivacy.PUBLIC,
 628         saveReplay
 629       }
 630
 631       const res = await createLive(servers[0].url, servers[0].accessToken, liveAttributes)
 632       return res.body.video.uuid
 633     }
 634
 635     before(async function () {
 636       this.timeout(60000)
 637
 638       liveVideoId = await createLiveWrapper(false)
 639       liveVideoReplayId = await createLiveWrapper(true)
 640
 641       await Promise.all([
 642         sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId),
 643         sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoReplayId)
 644       ])
 645
 646       await Promise.all([
 647         waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId),
 648         waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoReplayId)
 649       ])
 650
 651       await killallServers([ servers[0] ])
 652       await reRunServer(servers[0])
 653
 654       await wait(5000)
 655     })
 656
 657     it('Should cleanup lives', async function () {
 658       this.timeout(60000)
 659
 660       const res = await getVideo(servers[0].url, liveVideoId)
 661       const video: VideoDetails = res.body
 662
 663       expect(video.state.id).to.equal(VideoState.LIVE_ENDED)
 664     })
 665
 666     it('Should save a live replay', async function () {
 667       this.timeout(60000)
 668
 669       await waitJobs(servers)
 670
 671       const res = await getVideo(servers[0].url, liveVideoReplayId)
 672       const video: VideoDetails = res.body
 673
 674       expect(video.state.id).to.equal(VideoState.PUBLISHED)
 675     })
 676   })
 677
 678   after(async function () {
 679     await cleanupTests(servers)
 680   })
 681 })