server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { HttpStatusCode } from '@shared/core-utils'
   6 import {
   7   cleanupTests,
   8   closeAllSequelize,
   9   CommentsCommand,
  10   completeVideoCheck,
  11   flushAndRunMultipleServers,
  12   getVideo,
  13   getVideosList,
  14   immutableAssign,
  15   killallServers,
  16   reRunServer,
  17   ServerInfo,
  18   setAccessTokensToServers,
  19   setActorFollowScores,
  20   updateVideo,
  21   uploadVideo,
  22   uploadVideoAndGetId,
  23   wait,
  24   waitJobs
  25 } from '@shared/extra-utils'
  26 import { JobState, Video, VideoPrivacy } from '@shared/models'
  27
  28 const expect = chai.expect
  29
  30 describe('Test handle downs', function () {
  31   let servers: ServerInfo[] = []
  32   let threadIdServer1: number
  33   let threadIdServer2: number
  34   let commentIdServer1: number
  35   let commentIdServer2: number
  36   let missedVideo1: Video
  37   let missedVideo2: Video
  38   let unlistedVideo: Video
  39
  40   const videoIdsServer1: string[] = []
  41
  42   const videoAttributes = {
  43     name: 'my super name for server 1',
  44     category: 5,
  45     licence: 4,
  46     language: 'ja',
  47     nsfw: true,
  48     privacy: VideoPrivacy.PUBLIC,
  49     description: 'my super description for server 1',
  50     support: 'my super support text for server 1',
  51     tags: [ 'tag1p1', 'tag2p1' ],
  52     fixture: 'video_short1.webm'
  53   }
  54
  55   const unlistedVideoAttributes = immutableAssign(videoAttributes, {
  56     privacy: VideoPrivacy.UNLISTED
  57   })
  58
  59   let checkAttributes: any
  60   let unlistedCheckAttributes: any
  61
  62   let commentCommands: CommentsCommand[]
  63
  64   before(async function () {
  65     this.timeout(30000)
  66
  67     servers = await flushAndRunMultipleServers(3)
  68     commentCommands = servers.map(s => s.commentsCommand)
  69
  70     checkAttributes = {
  71       name: 'my super name for server 1',
  72       category: 5,
  73       licence: 4,
  74       language: 'ja',
  75       nsfw: true,
  76       description: 'my super description for server 1',
  77       support: 'my super support text for server 1',
  78       account: {
  79         name: 'root',
  80         host: 'localhost:' + servers[0].port
  81       },
  82       isLocal: false,
  83       duration: 10,
  84       tags: [ 'tag1p1', 'tag2p1' ],
  85       privacy: VideoPrivacy.PUBLIC,
  86       commentsEnabled: true,
  87       downloadEnabled: true,
  88       channel: {
  89         name: 'root_channel',
  90         displayName: 'Main root channel',
  91         description: '',
  92         isLocal: false
  93       },
  94       fixture: 'video_short1.webm',
  95       files: [
  96         {
  97           resolution: 720,
  98           size: 572456
  99         }
 100       ]
 101     }
 102     unlistedCheckAttributes = immutableAssign(checkAttributes, {
 103       privacy: VideoPrivacy.UNLISTED
 104     })
 105
 106     // Get the access tokens
 107     await setAccessTokensToServers(servers)
 108   })
 109
 110   it('Should remove followers that are often down', async function () {
 111     this.timeout(240000)
 112
 113     // Server 2 and 3 follow server 1
 114     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 115     await servers[2].followsCommand.follow({ targets: [ servers[0].url ] })
 116
 117     await waitJobs(servers)
 118
 119     // Upload a video to server 1
 120     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 121
 122     await waitJobs(servers)
 123
 124     // And check all servers have this video
 125     for (const server of servers) {
 126       const res = await getVideosList(server.url)
 127       expect(res.body.data).to.be.an('array')
 128       expect(res.body.data).to.have.lengthOf(1)
 129     }
 130
 131     // Kill server 2
 132     killallServers([ servers[1] ])
 133
 134     // Remove server 2 follower
 135     for (let i = 0; i < 10; i++) {
 136       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 137     }
 138
 139     await waitJobs([ servers[0], servers[2] ])
 140
 141     // Kill server 3
 142     killallServers([ servers[2] ])
 143
 144     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 145     missedVideo1 = resLastVideo1.body.video
 146
 147     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 148     missedVideo2 = resLastVideo2.body.video
 149
 150     // Unlisted video
 151     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 152     unlistedVideo = resVideo.body.video
 153
 154     // Add comments to video 2
 155     {
 156       const text = 'thread 1'
 157       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 158       threadIdServer1 = comment.id
 159
 160       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 161
 162       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 163       commentIdServer1 = created.id
 164     }
 165
 166     await waitJobs(servers[0])
 167     // Wait scheduler
 168     await wait(11000)
 169
 170     // Only server 3 is still a follower of server 1
 171     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 172     expect(body.data).to.be.an('array')
 173     expect(body.data).to.have.lengthOf(1)
 174     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 175   })
 176
 177   it('Should not have pending/processing jobs anymore', async function () {
 178     const states: JobState[] = [ 'waiting', 'active' ]
 179
 180     for (const state of states) {
 181       const body = await servers[0].jobsCommand.getJobsList({
 182         state: state,
 183         start: 0,
 184         count: 50,
 185         sort: '-createdAt'
 186       })
 187       expect(body.data).to.have.length(0)
 188     }
 189   })
 190
 191   it('Should re-follow server 1', async function () {
 192     this.timeout(35000)
 193
 194     await reRunServer(servers[1])
 195     await reRunServer(servers[2])
 196
 197     await servers[1].followsCommand.unfollow({ target: servers[0] })
 198     await waitJobs(servers)
 199
 200     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 201
 202     await waitJobs(servers)
 203
 204     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 205     expect(body.data).to.be.an('array')
 206     expect(body.data).to.have.lengthOf(2)
 207   })
 208
 209   it('Should send an update to server 3, and automatically fetch the video', async function () {
 210     this.timeout(15000)
 211
 212     const res1 = await getVideosList(servers[2].url)
 213     expect(res1.body.data).to.be.an('array')
 214     expect(res1.body.data).to.have.lengthOf(11)
 215
 216     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 217     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 218
 219     await waitJobs(servers)
 220
 221     const res = await getVideosList(servers[2].url)
 222     expect(res.body.data).to.be.an('array')
 223     // 1 video is unlisted
 224     expect(res.body.data).to.have.lengthOf(12)
 225
 226     // Check unlisted video
 227     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 228     expect(resVideo.body).not.to.be.undefined
 229
 230     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 231   })
 232
 233   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 234     this.timeout(25000)
 235
 236     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 237
 238     await waitJobs(servers)
 239
 240     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 241     expect(resVideo.body).not.to.be.undefined
 242
 243     {
 244       const { data } = await servers[2].commentsCommand.listThreads({ videoId: missedVideo2.uuid })
 245       expect(data).to.be.an('array')
 246       expect(data).to.have.lengthOf(1)
 247
 248       threadIdServer2 = data[0].id
 249
 250       const tree = await servers[2].commentsCommand.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 251       expect(tree.comment.text).equal('thread 1')
 252       expect(tree.children).to.have.lengthOf(1)
 253
 254       const firstChild = tree.children[0]
 255       expect(firstChild.comment.text).to.equal('comment 1-1')
 256       expect(firstChild.children).to.have.lengthOf(1)
 257
 258       const childOfFirstChild = firstChild.children[0]
 259       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 260       expect(childOfFirstChild.children).to.have.lengthOf(1)
 261
 262       const childOfChildFirstChild = childOfFirstChild.children[0]
 263       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 264       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 265
 266       commentIdServer2 = childOfChildFirstChild.comment.id
 267     }
 268   })
 269
 270   it('Should correctly reply to the comment', async function () {
 271     this.timeout(15000)
 272
 273     await servers[2].commentsCommand.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 274
 275     await waitJobs(servers)
 276
 277     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 278
 279     expect(tree.comment.text).equal('thread 1')
 280     expect(tree.children).to.have.lengthOf(1)
 281
 282     const firstChild = tree.children[0]
 283     expect(firstChild.comment.text).to.equal('comment 1-1')
 284     expect(firstChild.children).to.have.lengthOf(1)
 285
 286     const childOfFirstChild = firstChild.children[0]
 287     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 288     expect(childOfFirstChild.children).to.have.lengthOf(1)
 289
 290     const childOfChildFirstChild = childOfFirstChild.children[0]
 291     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 292     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 293
 294     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 295     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 296     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 297   })
 298
 299   it('Should upload many videos on server 1', async function () {
 300     this.timeout(120000)
 301
 302     for (let i = 0; i < 10; i++) {
 303       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 304       videoIdsServer1.push(uuid)
 305     }
 306
 307     await waitJobs(servers)
 308
 309     for (const id of videoIdsServer1) {
 310       await getVideo(servers[1].url, id)
 311     }
 312
 313     await waitJobs(servers)
 314     await setActorFollowScores(servers[1].internalServerNumber, 20)
 315
 316     // Wait video expiration
 317     await wait(11000)
 318
 319     // Refresh video -> score + 10 = 30
 320     await getVideo(servers[1].url, videoIdsServer1[0])
 321
 322     await waitJobs(servers)
 323   })
 324
 325   it('Should remove followings that are down', async function () {
 326     this.timeout(120000)
 327
 328     killallServers([ servers[0] ])
 329
 330     // Wait video expiration
 331     await wait(11000)
 332
 333     for (let i = 0; i < 5; i++) {
 334       try {
 335         await getVideo(servers[1].url, videoIdsServer1[i])
 336         await waitJobs([ servers[1] ])
 337         await wait(1500)
 338       } catch {}
 339     }
 340
 341     for (const id of videoIdsServer1) {
 342       await getVideo(servers[1].url, id, HttpStatusCode.FORBIDDEN_403)
 343     }
 344   })
 345
 346   after(async function () {
 347     await closeAllSequelize([ servers[1] ])
 348
 349     await cleanupTests(servers)
 350   })
 351 })