server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { HttpStatusCode } from '@shared/core-utils'
   6 import {
   7   cleanupTests,
   8   CommentsCommand,
   9   completeVideoCheck,
  10   flushAndRunMultipleServers,
  11   killallServers,
  12   reRunServer,
  13   ServerInfo,
  14   setAccessTokensToServers,
  15   wait,
  16   waitJobs
  17 } from '@shared/extra-utils'
  18 import { JobState, VideoCreateResult, VideoPrivacy } from '@shared/models'
  19
  20 const expect = chai.expect
  21
  22 describe('Test handle downs', function () {
  23   let servers: ServerInfo[] = []
  24   let threadIdServer1: number
  25   let threadIdServer2: number
  26   let commentIdServer1: number
  27   let commentIdServer2: number
  28   let missedVideo1: VideoCreateResult
  29   let missedVideo2: VideoCreateResult
  30   let unlistedVideo: VideoCreateResult
  31
  32   const videoIdsServer1: string[] = []
  33
  34   const videoAttributes = {
  35     name: 'my super name for server 1',
  36     category: 5,
  37     licence: 4,
  38     language: 'ja',
  39     nsfw: true,
  40     privacy: VideoPrivacy.PUBLIC,
  41     description: 'my super description for server 1',
  42     support: 'my super support text for server 1',
  43     tags: [ 'tag1p1', 'tag2p1' ],
  44     fixture: 'video_short1.webm'
  45   }
  46
  47   const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED }
  48
  49   let checkAttributes: any
  50   let unlistedCheckAttributes: any
  51
  52   let commentCommands: CommentsCommand[]
  53
  54   before(async function () {
  55     this.timeout(30000)
  56
  57     servers = await flushAndRunMultipleServers(3)
  58     commentCommands = servers.map(s => s.comments)
  59
  60     checkAttributes = {
  61       name: 'my super name for server 1',
  62       category: 5,
  63       licence: 4,
  64       language: 'ja',
  65       nsfw: true,
  66       description: 'my super description for server 1',
  67       support: 'my super support text for server 1',
  68       account: {
  69         name: 'root',
  70         host: 'localhost:' + servers[0].port
  71       },
  72       isLocal: false,
  73       duration: 10,
  74       tags: [ 'tag1p1', 'tag2p1' ],
  75       privacy: VideoPrivacy.PUBLIC,
  76       commentsEnabled: true,
  77       downloadEnabled: true,
  78       channel: {
  79         name: 'root_channel',
  80         displayName: 'Main root channel',
  81         description: '',
  82         isLocal: false
  83       },
  84       fixture: 'video_short1.webm',
  85       files: [
  86         {
  87           resolution: 720,
  88           size: 572456
  89         }
  90       ]
  91     }
  92     unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED }
  93
  94     // Get the access tokens
  95     await setAccessTokensToServers(servers)
  96   })
  97
  98   it('Should remove followers that are often down', async function () {
  99     this.timeout(240000)
 100
 101     // Server 2 and 3 follow server 1
 102     await servers[1].follows.follow({ targets: [ servers[0].url ] })
 103     await servers[2].follows.follow({ targets: [ servers[0].url ] })
 104
 105     await waitJobs(servers)
 106
 107     // Upload a video to server 1
 108     await servers[0].videos.upload({ attributes: videoAttributes })
 109
 110     await waitJobs(servers)
 111
 112     // And check all servers have this video
 113     for (const server of servers) {
 114       const { data } = await server.videos.list()
 115       expect(data).to.be.an('array')
 116       expect(data).to.have.lengthOf(1)
 117     }
 118
 119     // Kill server 2
 120     await killallServers([ servers[1] ])
 121
 122     // Remove server 2 follower
 123     for (let i = 0; i < 10; i++) {
 124       await servers[0].videos.upload({ attributes: videoAttributes })
 125     }
 126
 127     await waitJobs([ servers[0], servers[2] ])
 128
 129     // Kill server 3
 130     await killallServers([ servers[2] ])
 131
 132     missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes })
 133
 134     missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes })
 135
 136     // Unlisted video
 137     unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes })
 138
 139     // Add comments to video 2
 140     {
 141       const text = 'thread 1'
 142       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 143       threadIdServer1 = comment.id
 144
 145       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 146
 147       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 148       commentIdServer1 = created.id
 149     }
 150
 151     await waitJobs(servers[0])
 152     // Wait scheduler
 153     await wait(11000)
 154
 155     // Only server 3 is still a follower of server 1
 156     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 157     expect(body.data).to.be.an('array')
 158     expect(body.data).to.have.lengthOf(1)
 159     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 160   })
 161
 162   it('Should not have pending/processing jobs anymore', async function () {
 163     const states: JobState[] = [ 'waiting', 'active' ]
 164
 165     for (const state of states) {
 166       const body = await servers[0].jobs.getJobsList({
 167         state: state,
 168         start: 0,
 169         count: 50,
 170         sort: '-createdAt'
 171       })
 172       expect(body.data).to.have.length(0)
 173     }
 174   })
 175
 176   it('Should re-follow server 1', async function () {
 177     this.timeout(35000)
 178
 179     await reRunServer(servers[1])
 180     await reRunServer(servers[2])
 181
 182     await servers[1].follows.unfollow({ target: servers[0] })
 183     await waitJobs(servers)
 184
 185     await servers[1].follows.follow({ targets: [ servers[0].url ] })
 186
 187     await waitJobs(servers)
 188
 189     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 190     expect(body.data).to.be.an('array')
 191     expect(body.data).to.have.lengthOf(2)
 192   })
 193
 194   it('Should send an update to server 3, and automatically fetch the video', async function () {
 195     this.timeout(15000)
 196
 197     {
 198       const { data } = await servers[2].videos.list()
 199       expect(data).to.be.an('array')
 200       expect(data).to.have.lengthOf(11)
 201     }
 202
 203     await servers[0].videos.update({ id: missedVideo1.uuid })
 204     await servers[0].videos.update({ id: unlistedVideo.uuid })
 205
 206     await waitJobs(servers)
 207
 208     {
 209       const { data } = await servers[2].videos.list()
 210       expect(data).to.be.an('array')
 211       // 1 video is unlisted
 212       expect(data).to.have.lengthOf(12)
 213     }
 214
 215     // Check unlisted video
 216     const video = await servers[2].videos.get({ id: unlistedVideo.uuid })
 217     await completeVideoCheck(servers[2], video, unlistedCheckAttributes)
 218   })
 219
 220   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 221     this.timeout(25000)
 222
 223     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 224
 225     await waitJobs(servers)
 226
 227     await servers[2].videos.get({ id: missedVideo2.uuid })
 228
 229     {
 230       const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid })
 231       expect(data).to.be.an('array')
 232       expect(data).to.have.lengthOf(1)
 233
 234       threadIdServer2 = data[0].id
 235
 236       const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 237       expect(tree.comment.text).equal('thread 1')
 238       expect(tree.children).to.have.lengthOf(1)
 239
 240       const firstChild = tree.children[0]
 241       expect(firstChild.comment.text).to.equal('comment 1-1')
 242       expect(firstChild.children).to.have.lengthOf(1)
 243
 244       const childOfFirstChild = firstChild.children[0]
 245       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 246       expect(childOfFirstChild.children).to.have.lengthOf(1)
 247
 248       const childOfChildFirstChild = childOfFirstChild.children[0]
 249       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 250       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 251
 252       commentIdServer2 = childOfChildFirstChild.comment.id
 253     }
 254   })
 255
 256   it('Should correctly reply to the comment', async function () {
 257     this.timeout(15000)
 258
 259     await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 260
 261     await waitJobs(servers)
 262
 263     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 264
 265     expect(tree.comment.text).equal('thread 1')
 266     expect(tree.children).to.have.lengthOf(1)
 267
 268     const firstChild = tree.children[0]
 269     expect(firstChild.comment.text).to.equal('comment 1-1')
 270     expect(firstChild.children).to.have.lengthOf(1)
 271
 272     const childOfFirstChild = firstChild.children[0]
 273     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 274     expect(childOfFirstChild.children).to.have.lengthOf(1)
 275
 276     const childOfChildFirstChild = childOfFirstChild.children[0]
 277     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 278     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 279
 280     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 281     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 282     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 283   })
 284
 285   it('Should upload many videos on server 1', async function () {
 286     this.timeout(120000)
 287
 288     for (let i = 0; i < 10; i++) {
 289       const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid
 290       videoIdsServer1.push(uuid)
 291     }
 292
 293     await waitJobs(servers)
 294
 295     for (const id of videoIdsServer1) {
 296       await servers[1].videos.get({ id })
 297     }
 298
 299     await waitJobs(servers)
 300     await servers[1].sql.setActorFollowScores(20)
 301
 302     // Wait video expiration
 303     await wait(11000)
 304
 305     // Refresh video -> score + 10 = 30
 306     await servers[1].videos.get({ id: videoIdsServer1[0] })
 307
 308     await waitJobs(servers)
 309   })
 310
 311   it('Should remove followings that are down', async function () {
 312     this.timeout(120000)
 313
 314     await killallServers([ servers[0] ])
 315
 316     // Wait video expiration
 317     await wait(11000)
 318
 319     for (let i = 0; i < 5; i++) {
 320       try {
 321         await servers[1].videos.get({ id: videoIdsServer1[i] })
 322         await waitJobs([ servers[1] ])
 323         await wait(1500)
 324       } catch {}
 325     }
 326
 327     for (const id of videoIdsServer1) {
 328       await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 329     }
 330   })
 331
 332   after(async function () {
 333     await cleanupTests(servers)
 334   })
 335 })