server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { completeVideoCheck, SQLCommand } from '@server/tests/shared'
   5 import { wait } from '@shared/core-utils'
   6 import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models'
   7 import {
   8   cleanupTests,
   9   CommentsCommand,
  10   createMultipleServers,
  11   killallServers,
  12   PeerTubeServer,
  13   setAccessTokensToServers,
  14   waitJobs
  15 } from '@shared/server-commands'
  16
  17 describe('Test handle downs', function () {
  18   let servers: PeerTubeServer[] = []
  19   let sqlCommands: SQLCommand[]
  20
  21   let threadIdServer1: number
  22   let threadIdServer2: number
  23   let commentIdServer1: number
  24   let commentIdServer2: number
  25   let missedVideo1: VideoCreateResult
  26   let missedVideo2: VideoCreateResult
  27   let unlistedVideo: VideoCreateResult
  28
  29   const videoIdsServer1: string[] = []
  30
  31   const videoAttributes = {
  32     name: 'my super name for server 1',
  33     category: 5,
  34     licence: 4,
  35     language: 'ja',
  36     nsfw: true,
  37     privacy: VideoPrivacy.PUBLIC,
  38     description: 'my super description for server 1',
  39     support: 'my super support text for server 1',
  40     tags: [ 'tag1p1', 'tag2p1' ],
  41     fixture: 'video_short1.webm'
  42   }
  43
  44   const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED }
  45
  46   let checkAttributes: any
  47   let unlistedCheckAttributes: any
  48
  49   let commentCommands: CommentsCommand[]
  50
  51   before(async function () {
  52     this.timeout(120000)
  53
  54     servers = await createMultipleServers(3)
  55     commentCommands = servers.map(s => s.comments)
  56
  57     checkAttributes = {
  58       name: 'my super name for server 1',
  59       category: 5,
  60       licence: 4,
  61       language: 'ja',
  62       nsfw: true,
  63       description: 'my super description for server 1',
  64       support: 'my super support text for server 1',
  65       account: {
  66         name: 'root',
  67         host: servers[0].host
  68       },
  69       isLocal: false,
  70       duration: 10,
  71       tags: [ 'tag1p1', 'tag2p1' ],
  72       privacy: VideoPrivacy.PUBLIC,
  73       commentsEnabled: true,
  74       downloadEnabled: true,
  75       channel: {
  76         name: 'root_channel',
  77         displayName: 'Main root channel',
  78         description: '',
  79         isLocal: false
  80       },
  81       fixture: 'video_short1.webm',
  82       files: [
  83         {
  84           resolution: 720,
  85           size: 572456
  86         }
  87       ]
  88     }
  89     unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED }
  90
  91     // Get the access tokens
  92     await setAccessTokensToServers(servers)
  93
  94     sqlCommands = servers.map(s => new SQLCommand(s))
  95   })
  96
  97   it('Should remove followers that are often down', async function () {
  98     this.timeout(240000)
  99
 100     // Server 2 and 3 follow server 1
 101     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 102     await servers[2].follows.follow({ hosts: [ servers[0].url ] })
 103
 104     await waitJobs(servers)
 105
 106     // Upload a video to server 1
 107     await servers[0].videos.upload({ attributes: videoAttributes })
 108
 109     await waitJobs(servers)
 110
 111     // And check all servers have this video
 112     for (const server of servers) {
 113       const { data } = await server.videos.list()
 114       expect(data).to.be.an('array')
 115       expect(data).to.have.lengthOf(1)
 116     }
 117
 118     // Kill server 2
 119     await killallServers([ servers[1] ])
 120
 121     // Remove server 2 follower
 122     for (let i = 0; i < 10; i++) {
 123       await servers[0].videos.upload({ attributes: videoAttributes })
 124     }
 125
 126     await waitJobs([ servers[0], servers[2] ])
 127
 128     // Kill server 3
 129     await killallServers([ servers[2] ])
 130
 131     missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes })
 132
 133     missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes })
 134
 135     // Unlisted video
 136     unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes })
 137
 138     // Add comments to video 2
 139     {
 140       const text = 'thread 1'
 141       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 142       threadIdServer1 = comment.id
 143
 144       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 145
 146       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 147       commentIdServer1 = created.id
 148     }
 149
 150     await waitJobs(servers[0])
 151     // Wait scheduler
 152     await wait(11000)
 153
 154     // Only server 3 is still a follower of server 1
 155     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 156     expect(body.data).to.be.an('array')
 157     expect(body.data).to.have.lengthOf(1)
 158     expect(body.data[0].follower.host).to.equal(servers[2].host)
 159   })
 160
 161   it('Should not have pending/processing jobs anymore', async function () {
 162     const states: JobState[] = [ 'waiting', 'active' ]
 163
 164     for (const state of states) {
 165       const body = await servers[0].jobs.list({
 166         state,
 167         start: 0,
 168         count: 50,
 169         sort: '-createdAt'
 170       })
 171       expect(body.data).to.have.length(0)
 172     }
 173   })
 174
 175   it('Should re-follow server 1', async function () {
 176     this.timeout(70000)
 177
 178     await servers[1].run()
 179     await servers[2].run()
 180
 181     await servers[1].follows.unfollow({ target: servers[0] })
 182     await waitJobs(servers)
 183
 184     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 185
 186     await waitJobs(servers)
 187
 188     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 189     expect(body.data).to.be.an('array')
 190     expect(body.data).to.have.lengthOf(2)
 191   })
 192
 193   it('Should send an update to server 3, and automatically fetch the video', async function () {
 194     this.timeout(15000)
 195
 196     {
 197       const { data } = await servers[2].videos.list()
 198       expect(data).to.be.an('array')
 199       expect(data).to.have.lengthOf(11)
 200     }
 201
 202     await servers[0].videos.update({ id: missedVideo1.uuid })
 203     await servers[0].videos.update({ id: unlistedVideo.uuid })
 204
 205     await waitJobs(servers)
 206
 207     {
 208       const { data } = await servers[2].videos.list()
 209       expect(data).to.be.an('array')
 210       // 1 video is unlisted
 211       expect(data).to.have.lengthOf(12)
 212     }
 213
 214     // Check unlisted video
 215     const video = await servers[2].videos.get({ id: unlistedVideo.uuid })
 216     await completeVideoCheck({ server: servers[2], originServer: servers[0], videoUUID: video.uuid, attributes: unlistedCheckAttributes })
 217   })
 218
 219   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 220     this.timeout(25000)
 221
 222     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 223
 224     await waitJobs(servers)
 225
 226     await servers[2].videos.get({ id: missedVideo2.uuid })
 227
 228     {
 229       const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid })
 230       expect(data).to.be.an('array')
 231       expect(data).to.have.lengthOf(1)
 232
 233       threadIdServer2 = data[0].id
 234
 235       const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 236       expect(tree.comment.text).equal('thread 1')
 237       expect(tree.children).to.have.lengthOf(1)
 238
 239       const firstChild = tree.children[0]
 240       expect(firstChild.comment.text).to.equal('comment 1-1')
 241       expect(firstChild.children).to.have.lengthOf(1)
 242
 243       const childOfFirstChild = firstChild.children[0]
 244       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 245       expect(childOfFirstChild.children).to.have.lengthOf(1)
 246
 247       const childOfChildFirstChild = childOfFirstChild.children[0]
 248       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 249       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 250
 251       commentIdServer2 = childOfChildFirstChild.comment.id
 252     }
 253   })
 254
 255   it('Should correctly reply to the comment', async function () {
 256     this.timeout(15000)
 257
 258     await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 259
 260     await waitJobs(servers)
 261
 262     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 263
 264     expect(tree.comment.text).equal('thread 1')
 265     expect(tree.children).to.have.lengthOf(1)
 266
 267     const firstChild = tree.children[0]
 268     expect(firstChild.comment.text).to.equal('comment 1-1')
 269     expect(firstChild.children).to.have.lengthOf(1)
 270
 271     const childOfFirstChild = firstChild.children[0]
 272     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 273     expect(childOfFirstChild.children).to.have.lengthOf(1)
 274
 275     const childOfChildFirstChild = childOfFirstChild.children[0]
 276     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 277     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 278
 279     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 280     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 281     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 282   })
 283
 284   it('Should upload many videos on server 1', async function () {
 285     this.timeout(120000)
 286
 287     for (let i = 0; i < 10; i++) {
 288       const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid
 289       videoIdsServer1.push(uuid)
 290     }
 291
 292     await waitJobs(servers)
 293
 294     for (const id of videoIdsServer1) {
 295       await servers[1].videos.get({ id })
 296     }
 297
 298     await waitJobs(servers)
 299     await sqlCommands[1].setActorFollowScores(20)
 300
 301     // Wait video expiration
 302     await wait(11000)
 303
 304     // Refresh video -> score + 10 = 30
 305     await servers[1].videos.get({ id: videoIdsServer1[0] })
 306
 307     await waitJobs(servers)
 308   })
 309
 310   it('Should remove followings that are down', async function () {
 311     this.timeout(120000)
 312
 313     await killallServers([ servers[0] ])
 314
 315     // Wait video expiration
 316     await wait(11000)
 317
 318     for (let i = 0; i < 5; i++) {
 319       try {
 320         await servers[1].videos.get({ id: videoIdsServer1[i] })
 321         await waitJobs([ servers[1] ])
 322         await wait(1500)
 323       } catch {}
 324     }
 325
 326     for (const id of videoIdsServer1) {
 327       await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 328     }
 329   })
 330
 331   after(async function () {
 332     for (const sqlCommand of sqlCommands) {
 333       await sqlCommand.cleanup()
 334     }
 335
 336     await cleanupTests(servers)
 337   })
 338 })