server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import { completeVideoCheck } from '@server/tests/shared'
   5 import { wait } from '@shared/core-utils'
   6 import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models'
   7 import {
   8   cleanupTests,
   9   CommentsCommand,
  10   createMultipleServers,
  11   killallServers,
  12   PeerTubeServer,
  13   setAccessTokensToServers,
  14   waitJobs
  15 } from '@shared/server-commands'
  16
  17 const expect = chai.expect
  18
  19 describe('Test handle downs', function () {
  20   let servers: PeerTubeServer[] = []
  21   let threadIdServer1: number
  22   let threadIdServer2: number
  23   let commentIdServer1: number
  24   let commentIdServer2: number
  25   let missedVideo1: VideoCreateResult
  26   let missedVideo2: VideoCreateResult
  27   let unlistedVideo: VideoCreateResult
  28
  29   const videoIdsServer1: string[] = []
  30
  31   const videoAttributes = {
  32     name: 'my super name for server 1',
  33     category: 5,
  34     licence: 4,
  35     language: 'ja',
  36     nsfw: true,
  37     privacy: VideoPrivacy.PUBLIC,
  38     description: 'my super description for server 1',
  39     support: 'my super support text for server 1',
  40     tags: [ 'tag1p1', 'tag2p1' ],
  41     fixture: 'video_short1.webm'
  42   }
  43
  44   const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED }
  45
  46   let checkAttributes: any
  47   let unlistedCheckAttributes: any
  48
  49   let commentCommands: CommentsCommand[]
  50
  51   before(async function () {
  52     this.timeout(120000)
  53
  54     servers = await createMultipleServers(3)
  55     commentCommands = servers.map(s => s.comments)
  56
  57     checkAttributes = {
  58       name: 'my super name for server 1',
  59       category: 5,
  60       licence: 4,
  61       language: 'ja',
  62       nsfw: true,
  63       description: 'my super description for server 1',
  64       support: 'my super support text for server 1',
  65       account: {
  66         name: 'root',
  67         host: 'localhost:' + servers[0].port
  68       },
  69       isLocal: false,
  70       duration: 10,
  71       tags: [ 'tag1p1', 'tag2p1' ],
  72       privacy: VideoPrivacy.PUBLIC,
  73       commentsEnabled: true,
  74       downloadEnabled: true,
  75       channel: {
  76         name: 'root_channel',
  77         displayName: 'Main root channel',
  78         description: '',
  79         isLocal: false
  80       },
  81       fixture: 'video_short1.webm',
  82       files: [
  83         {
  84           resolution: 720,
  85           size: 572456
  86         }
  87       ]
  88     }
  89     unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED }
  90
  91     // Get the access tokens
  92     await setAccessTokensToServers(servers)
  93   })
  94
  95   it('Should remove followers that are often down', async function () {
  96     this.timeout(240000)
  97
  98     // Server 2 and 3 follow server 1
  99     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 100     await servers[2].follows.follow({ hosts: [ servers[0].url ] })
 101
 102     await waitJobs(servers)
 103
 104     // Upload a video to server 1
 105     await servers[0].videos.upload({ attributes: videoAttributes })
 106
 107     await waitJobs(servers)
 108
 109     // And check all servers have this video
 110     for (const server of servers) {
 111       const { data } = await server.videos.list()
 112       expect(data).to.be.an('array')
 113       expect(data).to.have.lengthOf(1)
 114     }
 115
 116     // Kill server 2
 117     await killallServers([ servers[1] ])
 118
 119     // Remove server 2 follower
 120     for (let i = 0; i < 10; i++) {
 121       await servers[0].videos.upload({ attributes: videoAttributes })
 122     }
 123
 124     await waitJobs([ servers[0], servers[2] ])
 125
 126     // Kill server 3
 127     await killallServers([ servers[2] ])
 128
 129     missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes })
 130
 131     missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes })
 132
 133     // Unlisted video
 134     unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes })
 135
 136     // Add comments to video 2
 137     {
 138       const text = 'thread 1'
 139       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 140       threadIdServer1 = comment.id
 141
 142       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 143
 144       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 145       commentIdServer1 = created.id
 146     }
 147
 148     await waitJobs(servers[0])
 149     // Wait scheduler
 150     await wait(11000)
 151
 152     // Only server 3 is still a follower of server 1
 153     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 154     expect(body.data).to.be.an('array')
 155     expect(body.data).to.have.lengthOf(1)
 156     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 157   })
 158
 159   it('Should not have pending/processing jobs anymore', async function () {
 160     const states: JobState[] = [ 'waiting', 'active' ]
 161
 162     for (const state of states) {
 163       const body = await servers[0].jobs.list({
 164         state,
 165         start: 0,
 166         count: 50,
 167         sort: '-createdAt'
 168       })
 169       expect(body.data).to.have.length(0)
 170     }
 171   })
 172
 173   it('Should re-follow server 1', async function () {
 174     this.timeout(70000)
 175
 176     await servers[1].run()
 177     await servers[2].run()
 178
 179     await servers[1].follows.unfollow({ target: servers[0] })
 180     await waitJobs(servers)
 181
 182     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 183
 184     await waitJobs(servers)
 185
 186     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 187     expect(body.data).to.be.an('array')
 188     expect(body.data).to.have.lengthOf(2)
 189   })
 190
 191   it('Should send an update to server 3, and automatically fetch the video', async function () {
 192     this.timeout(15000)
 193
 194     {
 195       const { data } = await servers[2].videos.list()
 196       expect(data).to.be.an('array')
 197       expect(data).to.have.lengthOf(11)
 198     }
 199
 200     await servers[0].videos.update({ id: missedVideo1.uuid })
 201     await servers[0].videos.update({ id: unlistedVideo.uuid })
 202
 203     await waitJobs(servers)
 204
 205     {
 206       const { data } = await servers[2].videos.list()
 207       expect(data).to.be.an('array')
 208       // 1 video is unlisted
 209       expect(data).to.have.lengthOf(12)
 210     }
 211
 212     // Check unlisted video
 213     const video = await servers[2].videos.get({ id: unlistedVideo.uuid })
 214     await completeVideoCheck(servers[2], video, unlistedCheckAttributes)
 215   })
 216
 217   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 218     this.timeout(25000)
 219
 220     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 221
 222     await waitJobs(servers)
 223
 224     await servers[2].videos.get({ id: missedVideo2.uuid })
 225
 226     {
 227       const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid })
 228       expect(data).to.be.an('array')
 229       expect(data).to.have.lengthOf(1)
 230
 231       threadIdServer2 = data[0].id
 232
 233       const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 234       expect(tree.comment.text).equal('thread 1')
 235       expect(tree.children).to.have.lengthOf(1)
 236
 237       const firstChild = tree.children[0]
 238       expect(firstChild.comment.text).to.equal('comment 1-1')
 239       expect(firstChild.children).to.have.lengthOf(1)
 240
 241       const childOfFirstChild = firstChild.children[0]
 242       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 243       expect(childOfFirstChild.children).to.have.lengthOf(1)
 244
 245       const childOfChildFirstChild = childOfFirstChild.children[0]
 246       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 247       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 248
 249       commentIdServer2 = childOfChildFirstChild.comment.id
 250     }
 251   })
 252
 253   it('Should correctly reply to the comment', async function () {
 254     this.timeout(15000)
 255
 256     await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 257
 258     await waitJobs(servers)
 259
 260     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 261
 262     expect(tree.comment.text).equal('thread 1')
 263     expect(tree.children).to.have.lengthOf(1)
 264
 265     const firstChild = tree.children[0]
 266     expect(firstChild.comment.text).to.equal('comment 1-1')
 267     expect(firstChild.children).to.have.lengthOf(1)
 268
 269     const childOfFirstChild = firstChild.children[0]
 270     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 271     expect(childOfFirstChild.children).to.have.lengthOf(1)
 272
 273     const childOfChildFirstChild = childOfFirstChild.children[0]
 274     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 275     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 276
 277     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 278     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 279     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 280   })
 281
 282   it('Should upload many videos on server 1', async function () {
 283     this.timeout(120000)
 284
 285     for (let i = 0; i < 10; i++) {
 286       const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid
 287       videoIdsServer1.push(uuid)
 288     }
 289
 290     await waitJobs(servers)
 291
 292     for (const id of videoIdsServer1) {
 293       await servers[1].videos.get({ id })
 294     }
 295
 296     await waitJobs(servers)
 297     await servers[1].sql.setActorFollowScores(20)
 298
 299     // Wait video expiration
 300     await wait(11000)
 301
 302     // Refresh video -> score + 10 = 30
 303     await servers[1].videos.get({ id: videoIdsServer1[0] })
 304
 305     await waitJobs(servers)
 306   })
 307
 308   it('Should remove followings that are down', async function () {
 309     this.timeout(120000)
 310
 311     await killallServers([ servers[0] ])
 312
 313     // Wait video expiration
 314     await wait(11000)
 315
 316     for (let i = 0; i < 5; i++) {
 317       try {
 318         await servers[1].videos.get({ id: videoIdsServer1[i] })
 319         await waitJobs([ servers[1] ])
 320         await wait(1500)
 321       } catch {}
 322     }
 323
 324     for (const id of videoIdsServer1) {
 325       await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 326     }
 327   })
 328
 329   after(async function () {
 330     await cleanupTests(servers)
 331   })
 332 })