server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { HttpStatusCode } from '@shared/core-utils'
   6 import {
   7   cleanupTests,
   8   CommentsCommand,
   9   completeVideoCheck,
  10   flushAndRunMultipleServers,
  11   getVideo,
  12   getVideosList,
  13   immutableAssign,
  14   killallServers,
  15   reRunServer,
  16   ServerInfo,
  17   setAccessTokensToServers,
  18   updateVideo,
  19   uploadVideo,
  20   uploadVideoAndGetId,
  21   wait,
  22   waitJobs
  23 } from '@shared/extra-utils'
  24 import { JobState, Video, VideoPrivacy } from '@shared/models'
  25
  26 const expect = chai.expect
  27
  28 describe('Test handle downs', function () {
  29   let servers: ServerInfo[] = []
  30   let threadIdServer1: number
  31   let threadIdServer2: number
  32   let commentIdServer1: number
  33   let commentIdServer2: number
  34   let missedVideo1: Video
  35   let missedVideo2: Video
  36   let unlistedVideo: Video
  37
  38   const videoIdsServer1: string[] = []
  39
  40   const videoAttributes = {
  41     name: 'my super name for server 1',
  42     category: 5,
  43     licence: 4,
  44     language: 'ja',
  45     nsfw: true,
  46     privacy: VideoPrivacy.PUBLIC,
  47     description: 'my super description for server 1',
  48     support: 'my super support text for server 1',
  49     tags: [ 'tag1p1', 'tag2p1' ],
  50     fixture: 'video_short1.webm'
  51   }
  52
  53   const unlistedVideoAttributes = immutableAssign(videoAttributes, {
  54     privacy: VideoPrivacy.UNLISTED
  55   })
  56
  57   let checkAttributes: any
  58   let unlistedCheckAttributes: any
  59
  60   let commentCommands: CommentsCommand[]
  61
  62   before(async function () {
  63     this.timeout(30000)
  64
  65     servers = await flushAndRunMultipleServers(3)
  66     commentCommands = servers.map(s => s.commentsCommand)
  67
  68     checkAttributes = {
  69       name: 'my super name for server 1',
  70       category: 5,
  71       licence: 4,
  72       language: 'ja',
  73       nsfw: true,
  74       description: 'my super description for server 1',
  75       support: 'my super support text for server 1',
  76       account: {
  77         name: 'root',
  78         host: 'localhost:' + servers[0].port
  79       },
  80       isLocal: false,
  81       duration: 10,
  82       tags: [ 'tag1p1', 'tag2p1' ],
  83       privacy: VideoPrivacy.PUBLIC,
  84       commentsEnabled: true,
  85       downloadEnabled: true,
  86       channel: {
  87         name: 'root_channel',
  88         displayName: 'Main root channel',
  89         description: '',
  90         isLocal: false
  91       },
  92       fixture: 'video_short1.webm',
  93       files: [
  94         {
  95           resolution: 720,
  96           size: 572456
  97         }
  98       ]
  99     }
 100     unlistedCheckAttributes = immutableAssign(checkAttributes, {
 101       privacy: VideoPrivacy.UNLISTED
 102     })
 103
 104     // Get the access tokens
 105     await setAccessTokensToServers(servers)
 106   })
 107
 108   it('Should remove followers that are often down', async function () {
 109     this.timeout(240000)
 110
 111     // Server 2 and 3 follow server 1
 112     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 113     await servers[2].followsCommand.follow({ targets: [ servers[0].url ] })
 114
 115     await waitJobs(servers)
 116
 117     // Upload a video to server 1
 118     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 119
 120     await waitJobs(servers)
 121
 122     // And check all servers have this video
 123     for (const server of servers) {
 124       const res = await getVideosList(server.url)
 125       expect(res.body.data).to.be.an('array')
 126       expect(res.body.data).to.have.lengthOf(1)
 127     }
 128
 129     // Kill server 2
 130     await killallServers([ servers[1] ])
 131
 132     // Remove server 2 follower
 133     for (let i = 0; i < 10; i++) {
 134       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 135     }
 136
 137     await waitJobs([ servers[0], servers[2] ])
 138
 139     // Kill server 3
 140     await killallServers([ servers[2] ])
 141
 142     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 143     missedVideo1 = resLastVideo1.body.video
 144
 145     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 146     missedVideo2 = resLastVideo2.body.video
 147
 148     // Unlisted video
 149     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 150     unlistedVideo = resVideo.body.video
 151
 152     // Add comments to video 2
 153     {
 154       const text = 'thread 1'
 155       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 156       threadIdServer1 = comment.id
 157
 158       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 159
 160       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 161       commentIdServer1 = created.id
 162     }
 163
 164     await waitJobs(servers[0])
 165     // Wait scheduler
 166     await wait(11000)
 167
 168     // Only server 3 is still a follower of server 1
 169     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 170     expect(body.data).to.be.an('array')
 171     expect(body.data).to.have.lengthOf(1)
 172     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 173   })
 174
 175   it('Should not have pending/processing jobs anymore', async function () {
 176     const states: JobState[] = [ 'waiting', 'active' ]
 177
 178     for (const state of states) {
 179       const body = await servers[0].jobsCommand.getJobsList({
 180         state: state,
 181         start: 0,
 182         count: 50,
 183         sort: '-createdAt'
 184       })
 185       expect(body.data).to.have.length(0)
 186     }
 187   })
 188
 189   it('Should re-follow server 1', async function () {
 190     this.timeout(35000)
 191
 192     await reRunServer(servers[1])
 193     await reRunServer(servers[2])
 194
 195     await servers[1].followsCommand.unfollow({ target: servers[0] })
 196     await waitJobs(servers)
 197
 198     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 199
 200     await waitJobs(servers)
 201
 202     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 203     expect(body.data).to.be.an('array')
 204     expect(body.data).to.have.lengthOf(2)
 205   })
 206
 207   it('Should send an update to server 3, and automatically fetch the video', async function () {
 208     this.timeout(15000)
 209
 210     const res1 = await getVideosList(servers[2].url)
 211     expect(res1.body.data).to.be.an('array')
 212     expect(res1.body.data).to.have.lengthOf(11)
 213
 214     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 215     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 216
 217     await waitJobs(servers)
 218
 219     const res = await getVideosList(servers[2].url)
 220     expect(res.body.data).to.be.an('array')
 221     // 1 video is unlisted
 222     expect(res.body.data).to.have.lengthOf(12)
 223
 224     // Check unlisted video
 225     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 226     expect(resVideo.body).not.to.be.undefined
 227
 228     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 229   })
 230
 231   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 232     this.timeout(25000)
 233
 234     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 235
 236     await waitJobs(servers)
 237
 238     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 239     expect(resVideo.body).not.to.be.undefined
 240
 241     {
 242       const { data } = await servers[2].commentsCommand.listThreads({ videoId: missedVideo2.uuid })
 243       expect(data).to.be.an('array')
 244       expect(data).to.have.lengthOf(1)
 245
 246       threadIdServer2 = data[0].id
 247
 248       const tree = await servers[2].commentsCommand.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 249       expect(tree.comment.text).equal('thread 1')
 250       expect(tree.children).to.have.lengthOf(1)
 251
 252       const firstChild = tree.children[0]
 253       expect(firstChild.comment.text).to.equal('comment 1-1')
 254       expect(firstChild.children).to.have.lengthOf(1)
 255
 256       const childOfFirstChild = firstChild.children[0]
 257       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 258       expect(childOfFirstChild.children).to.have.lengthOf(1)
 259
 260       const childOfChildFirstChild = childOfFirstChild.children[0]
 261       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 262       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 263
 264       commentIdServer2 = childOfChildFirstChild.comment.id
 265     }
 266   })
 267
 268   it('Should correctly reply to the comment', async function () {
 269     this.timeout(15000)
 270
 271     await servers[2].commentsCommand.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 272
 273     await waitJobs(servers)
 274
 275     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 276
 277     expect(tree.comment.text).equal('thread 1')
 278     expect(tree.children).to.have.lengthOf(1)
 279
 280     const firstChild = tree.children[0]
 281     expect(firstChild.comment.text).to.equal('comment 1-1')
 282     expect(firstChild.children).to.have.lengthOf(1)
 283
 284     const childOfFirstChild = firstChild.children[0]
 285     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 286     expect(childOfFirstChild.children).to.have.lengthOf(1)
 287
 288     const childOfChildFirstChild = childOfFirstChild.children[0]
 289     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 290     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 291
 292     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 293     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 294     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 295   })
 296
 297   it('Should upload many videos on server 1', async function () {
 298     this.timeout(120000)
 299
 300     for (let i = 0; i < 10; i++) {
 301       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 302       videoIdsServer1.push(uuid)
 303     }
 304
 305     await waitJobs(servers)
 306
 307     for (const id of videoIdsServer1) {
 308       await getVideo(servers[1].url, id)
 309     }
 310
 311     await waitJobs(servers)
 312     await servers[1].sqlCommand.setActorFollowScores(20)
 313
 314     // Wait video expiration
 315     await wait(11000)
 316
 317     // Refresh video -> score + 10 = 30
 318     await getVideo(servers[1].url, videoIdsServer1[0])
 319
 320     await waitJobs(servers)
 321   })
 322
 323   it('Should remove followings that are down', async function () {
 324     this.timeout(120000)
 325
 326     await killallServers([ servers[0] ])
 327
 328     // Wait video expiration
 329     await wait(11000)
 330
 331     for (let i = 0; i < 5; i++) {
 332       try {
 333         await getVideo(servers[1].url, videoIdsServer1[i])
 334         await waitJobs([ servers[1] ])
 335         await wait(1500)
 336       } catch {}
 337     }
 338
 339     for (const id of videoIdsServer1) {
 340       await getVideo(servers[1].url, id, HttpStatusCode.FORBIDDEN_403)
 341     }
 342   })
 343
 344   after(async function () {
 345     await cleanupTests(servers)
 346   })
 347 })