server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import {
   6   cleanupTests,
   7   CommentsCommand,
   8   completeVideoCheck,
   9   createMultipleServers,
  10   killallServers,
  11   PeerTubeServer,
  12   setAccessTokensToServers,
  13   wait,
  14   waitJobs
  15 } from '@shared/extra-utils'
  16 import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models'
  17
  18 const expect = chai.expect
  19
  20 describe('Test handle downs', function () {
  21   let servers: PeerTubeServer[] = []
  22   let threadIdServer1: number
  23   let threadIdServer2: number
  24   let commentIdServer1: number
  25   let commentIdServer2: number
  26   let missedVideo1: VideoCreateResult
  27   let missedVideo2: VideoCreateResult
  28   let unlistedVideo: VideoCreateResult
  29
  30   const videoIdsServer1: string[] = []
  31
  32   const videoAttributes = {
  33     name: 'my super name for server 1',
  34     category: 5,
  35     licence: 4,
  36     language: 'ja',
  37     nsfw: true,
  38     privacy: VideoPrivacy.PUBLIC,
  39     description: 'my super description for server 1',
  40     support: 'my super support text for server 1',
  41     tags: [ 'tag1p1', 'tag2p1' ],
  42     fixture: 'video_short1.webm'
  43   }
  44
  45   const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED }
  46
  47   let checkAttributes: any
  48   let unlistedCheckAttributes: any
  49
  50   let commentCommands: CommentsCommand[]
  51
  52   before(async function () {
  53     this.timeout(30000)
  54
  55     servers = await createMultipleServers(3)
  56     commentCommands = servers.map(s => s.comments)
  57
  58     checkAttributes = {
  59       name: 'my super name for server 1',
  60       category: 5,
  61       licence: 4,
  62       language: 'ja',
  63       nsfw: true,
  64       description: 'my super description for server 1',
  65       support: 'my super support text for server 1',
  66       account: {
  67         name: 'root',
  68         host: 'localhost:' + servers[0].port
  69       },
  70       isLocal: false,
  71       duration: 10,
  72       tags: [ 'tag1p1', 'tag2p1' ],
  73       privacy: VideoPrivacy.PUBLIC,
  74       commentsEnabled: true,
  75       downloadEnabled: true,
  76       channel: {
  77         name: 'root_channel',
  78         displayName: 'Main root channel',
  79         description: '',
  80         isLocal: false
  81       },
  82       fixture: 'video_short1.webm',
  83       files: [
  84         {
  85           resolution: 720,
  86           size: 572456
  87         }
  88       ]
  89     }
  90     unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED }
  91
  92     // Get the access tokens
  93     await setAccessTokensToServers(servers)
  94   })
  95
  96   it('Should remove followers that are often down', async function () {
  97     this.timeout(240000)
  98
  99     // Server 2 and 3 follow server 1
 100     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 101     await servers[2].follows.follow({ hosts: [ servers[0].url ] })
 102
 103     await waitJobs(servers)
 104
 105     // Upload a video to server 1
 106     await servers[0].videos.upload({ attributes: videoAttributes })
 107
 108     await waitJobs(servers)
 109
 110     // And check all servers have this video
 111     for (const server of servers) {
 112       const { data } = await server.videos.list()
 113       expect(data).to.be.an('array')
 114       expect(data).to.have.lengthOf(1)
 115     }
 116
 117     // Kill server 2
 118     await killallServers([ servers[1] ])
 119
 120     // Remove server 2 follower
 121     for (let i = 0; i < 10; i++) {
 122       await servers[0].videos.upload({ attributes: videoAttributes })
 123     }
 124
 125     await waitJobs([ servers[0], servers[2] ])
 126
 127     // Kill server 3
 128     await killallServers([ servers[2] ])
 129
 130     missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes })
 131
 132     missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes })
 133
 134     // Unlisted video
 135     unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes })
 136
 137     // Add comments to video 2
 138     {
 139       const text = 'thread 1'
 140       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 141       threadIdServer1 = comment.id
 142
 143       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 144
 145       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 146       commentIdServer1 = created.id
 147     }
 148
 149     await waitJobs(servers[0])
 150     // Wait scheduler
 151     await wait(11000)
 152
 153     // Only server 3 is still a follower of server 1
 154     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 155     expect(body.data).to.be.an('array')
 156     expect(body.data).to.have.lengthOf(1)
 157     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 158   })
 159
 160   it('Should not have pending/processing jobs anymore', async function () {
 161     const states: JobState[] = [ 'waiting', 'active' ]
 162
 163     for (const state of states) {
 164       const body = await servers[0].jobs.getJobsList({
 165         state: state,
 166         start: 0,
 167         count: 50,
 168         sort: '-createdAt'
 169       })
 170       expect(body.data).to.have.length(0)
 171     }
 172   })
 173
 174   it('Should re-follow server 1', async function () {
 175     this.timeout(35000)
 176
 177     await servers[1].run()
 178     await servers[2].run()
 179
 180     await servers[1].follows.unfollow({ target: servers[0] })
 181     await waitJobs(servers)
 182
 183     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 184
 185     await waitJobs(servers)
 186
 187     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 188     expect(body.data).to.be.an('array')
 189     expect(body.data).to.have.lengthOf(2)
 190   })
 191
 192   it('Should send an update to server 3, and automatically fetch the video', async function () {
 193     this.timeout(15000)
 194
 195     {
 196       const { data } = await servers[2].videos.list()
 197       expect(data).to.be.an('array')
 198       expect(data).to.have.lengthOf(11)
 199     }
 200
 201     await servers[0].videos.update({ id: missedVideo1.uuid })
 202     await servers[0].videos.update({ id: unlistedVideo.uuid })
 203
 204     await waitJobs(servers)
 205
 206     {
 207       const { data } = await servers[2].videos.list()
 208       expect(data).to.be.an('array')
 209       // 1 video is unlisted
 210       expect(data).to.have.lengthOf(12)
 211     }
 212
 213     // Check unlisted video
 214     const video = await servers[2].videos.get({ id: unlistedVideo.uuid })
 215     await completeVideoCheck(servers[2], video, unlistedCheckAttributes)
 216   })
 217
 218   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 219     this.timeout(25000)
 220
 221     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 222
 223     await waitJobs(servers)
 224
 225     await servers[2].videos.get({ id: missedVideo2.uuid })
 226
 227     {
 228       const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid })
 229       expect(data).to.be.an('array')
 230       expect(data).to.have.lengthOf(1)
 231
 232       threadIdServer2 = data[0].id
 233
 234       const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 235       expect(tree.comment.text).equal('thread 1')
 236       expect(tree.children).to.have.lengthOf(1)
 237
 238       const firstChild = tree.children[0]
 239       expect(firstChild.comment.text).to.equal('comment 1-1')
 240       expect(firstChild.children).to.have.lengthOf(1)
 241
 242       const childOfFirstChild = firstChild.children[0]
 243       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 244       expect(childOfFirstChild.children).to.have.lengthOf(1)
 245
 246       const childOfChildFirstChild = childOfFirstChild.children[0]
 247       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 248       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 249
 250       commentIdServer2 = childOfChildFirstChild.comment.id
 251     }
 252   })
 253
 254   it('Should correctly reply to the comment', async function () {
 255     this.timeout(15000)
 256
 257     await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 258
 259     await waitJobs(servers)
 260
 261     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 262
 263     expect(tree.comment.text).equal('thread 1')
 264     expect(tree.children).to.have.lengthOf(1)
 265
 266     const firstChild = tree.children[0]
 267     expect(firstChild.comment.text).to.equal('comment 1-1')
 268     expect(firstChild.children).to.have.lengthOf(1)
 269
 270     const childOfFirstChild = firstChild.children[0]
 271     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 272     expect(childOfFirstChild.children).to.have.lengthOf(1)
 273
 274     const childOfChildFirstChild = childOfFirstChild.children[0]
 275     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 276     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 277
 278     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 279     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 280     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 281   })
 282
 283   it('Should upload many videos on server 1', async function () {
 284     this.timeout(120000)
 285
 286     for (let i = 0; i < 10; i++) {
 287       const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid
 288       videoIdsServer1.push(uuid)
 289     }
 290
 291     await waitJobs(servers)
 292
 293     for (const id of videoIdsServer1) {
 294       await servers[1].videos.get({ id })
 295     }
 296
 297     await waitJobs(servers)
 298     await servers[1].sql.setActorFollowScores(20)
 299
 300     // Wait video expiration
 301     await wait(11000)
 302
 303     // Refresh video -> score + 10 = 30
 304     await servers[1].videos.get({ id: videoIdsServer1[0] })
 305
 306     await waitJobs(servers)
 307   })
 308
 309   it('Should remove followings that are down', async function () {
 310     this.timeout(120000)
 311
 312     await killallServers([ servers[0] ])
 313
 314     // Wait video expiration
 315     await wait(11000)
 316
 317     for (let i = 0; i < 5; i++) {
 318       try {
 319         await servers[1].videos.get({ id: videoIdsServer1[i] })
 320         await waitJobs([ servers[1] ])
 321         await wait(1500)
 322       } catch {}
 323     }
 324
 325     for (const id of videoIdsServer1) {
 326       await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 327     }
 328   })
 329
 330   after(async function () {
 331     await cleanupTests(servers)
 332   })
 333 })