server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { completeVideoCheck } from '@server/tests/shared'
   5 import { wait } from '@shared/core-utils'
   6 import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models'
   7 import {
   8   cleanupTests,
   9   CommentsCommand,
  10   createMultipleServers,
  11   killallServers,
  12   PeerTubeServer,
  13   setAccessTokensToServers,
  14   waitJobs
  15 } from '@shared/server-commands'
  16
  17 describe('Test handle downs', function () {
  18   let servers: PeerTubeServer[] = []
  19   let threadIdServer1: number
  20   let threadIdServer2: number
  21   let commentIdServer1: number
  22   let commentIdServer2: number
  23   let missedVideo1: VideoCreateResult
  24   let missedVideo2: VideoCreateResult
  25   let unlistedVideo: VideoCreateResult
  26
  27   const videoIdsServer1: string[] = []
  28
  29   const videoAttributes = {
  30     name: 'my super name for server 1',
  31     category: 5,
  32     licence: 4,
  33     language: 'ja',
  34     nsfw: true,
  35     privacy: VideoPrivacy.PUBLIC,
  36     description: 'my super description for server 1',
  37     support: 'my super support text for server 1',
  38     tags: [ 'tag1p1', 'tag2p1' ],
  39     fixture: 'video_short1.webm'
  40   }
  41
  42   const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED }
  43
  44   let checkAttributes: any
  45   let unlistedCheckAttributes: any
  46
  47   let commentCommands: CommentsCommand[]
  48
  49   before(async function () {
  50     this.timeout(120000)
  51
  52     servers = await createMultipleServers(3)
  53     commentCommands = servers.map(s => s.comments)
  54
  55     checkAttributes = {
  56       name: 'my super name for server 1',
  57       category: 5,
  58       licence: 4,
  59       language: 'ja',
  60       nsfw: true,
  61       description: 'my super description for server 1',
  62       support: 'my super support text for server 1',
  63       account: {
  64         name: 'root',
  65         host: 'localhost:' + servers[0].port
  66       },
  67       isLocal: false,
  68       duration: 10,
  69       tags: [ 'tag1p1', 'tag2p1' ],
  70       privacy: VideoPrivacy.PUBLIC,
  71       commentsEnabled: true,
  72       downloadEnabled: true,
  73       channel: {
  74         name: 'root_channel',
  75         displayName: 'Main root channel',
  76         description: '',
  77         isLocal: false
  78       },
  79       fixture: 'video_short1.webm',
  80       files: [
  81         {
  82           resolution: 720,
  83           size: 572456
  84         }
  85       ]
  86     }
  87     unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED }
  88
  89     // Get the access tokens
  90     await setAccessTokensToServers(servers)
  91   })
  92
  93   it('Should remove followers that are often down', async function () {
  94     this.timeout(240000)
  95
  96     // Server 2 and 3 follow server 1
  97     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
  98     await servers[2].follows.follow({ hosts: [ servers[0].url ] })
  99
 100     await waitJobs(servers)
 101
 102     // Upload a video to server 1
 103     await servers[0].videos.upload({ attributes: videoAttributes })
 104
 105     await waitJobs(servers)
 106
 107     // And check all servers have this video
 108     for (const server of servers) {
 109       const { data } = await server.videos.list()
 110       expect(data).to.be.an('array')
 111       expect(data).to.have.lengthOf(1)
 112     }
 113
 114     // Kill server 2
 115     await killallServers([ servers[1] ])
 116
 117     // Remove server 2 follower
 118     for (let i = 0; i < 10; i++) {
 119       await servers[0].videos.upload({ attributes: videoAttributes })
 120     }
 121
 122     await waitJobs([ servers[0], servers[2] ])
 123
 124     // Kill server 3
 125     await killallServers([ servers[2] ])
 126
 127     missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes })
 128
 129     missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes })
 130
 131     // Unlisted video
 132     unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes })
 133
 134     // Add comments to video 2
 135     {
 136       const text = 'thread 1'
 137       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 138       threadIdServer1 = comment.id
 139
 140       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 141
 142       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 143       commentIdServer1 = created.id
 144     }
 145
 146     await waitJobs(servers[0])
 147     // Wait scheduler
 148     await wait(11000)
 149
 150     // Only server 3 is still a follower of server 1
 151     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 152     expect(body.data).to.be.an('array')
 153     expect(body.data).to.have.lengthOf(1)
 154     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 155   })
 156
 157   it('Should not have pending/processing jobs anymore', async function () {
 158     const states: JobState[] = [ 'waiting', 'active' ]
 159
 160     for (const state of states) {
 161       const body = await servers[0].jobs.list({
 162         state,
 163         start: 0,
 164         count: 50,
 165         sort: '-createdAt'
 166       })
 167       expect(body.data).to.have.length(0)
 168     }
 169   })
 170
 171   it('Should re-follow server 1', async function () {
 172     this.timeout(70000)
 173
 174     await servers[1].run()
 175     await servers[2].run()
 176
 177     await servers[1].follows.unfollow({ target: servers[0] })
 178     await waitJobs(servers)
 179
 180     await servers[1].follows.follow({ hosts: [ servers[0].url ] })
 181
 182     await waitJobs(servers)
 183
 184     const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 185     expect(body.data).to.be.an('array')
 186     expect(body.data).to.have.lengthOf(2)
 187   })
 188
 189   it('Should send an update to server 3, and automatically fetch the video', async function () {
 190     this.timeout(15000)
 191
 192     {
 193       const { data } = await servers[2].videos.list()
 194       expect(data).to.be.an('array')
 195       expect(data).to.have.lengthOf(11)
 196     }
 197
 198     await servers[0].videos.update({ id: missedVideo1.uuid })
 199     await servers[0].videos.update({ id: unlistedVideo.uuid })
 200
 201     await waitJobs(servers)
 202
 203     {
 204       const { data } = await servers[2].videos.list()
 205       expect(data).to.be.an('array')
 206       // 1 video is unlisted
 207       expect(data).to.have.lengthOf(12)
 208     }
 209
 210     // Check unlisted video
 211     const video = await servers[2].videos.get({ id: unlistedVideo.uuid })
 212     await completeVideoCheck(servers[2], video, unlistedCheckAttributes)
 213   })
 214
 215   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 216     this.timeout(25000)
 217
 218     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 219
 220     await waitJobs(servers)
 221
 222     await servers[2].videos.get({ id: missedVideo2.uuid })
 223
 224     {
 225       const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid })
 226       expect(data).to.be.an('array')
 227       expect(data).to.have.lengthOf(1)
 228
 229       threadIdServer2 = data[0].id
 230
 231       const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 232       expect(tree.comment.text).equal('thread 1')
 233       expect(tree.children).to.have.lengthOf(1)
 234
 235       const firstChild = tree.children[0]
 236       expect(firstChild.comment.text).to.equal('comment 1-1')
 237       expect(firstChild.children).to.have.lengthOf(1)
 238
 239       const childOfFirstChild = firstChild.children[0]
 240       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 241       expect(childOfFirstChild.children).to.have.lengthOf(1)
 242
 243       const childOfChildFirstChild = childOfFirstChild.children[0]
 244       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 245       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 246
 247       commentIdServer2 = childOfChildFirstChild.comment.id
 248     }
 249   })
 250
 251   it('Should correctly reply to the comment', async function () {
 252     this.timeout(15000)
 253
 254     await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 255
 256     await waitJobs(servers)
 257
 258     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 259
 260     expect(tree.comment.text).equal('thread 1')
 261     expect(tree.children).to.have.lengthOf(1)
 262
 263     const firstChild = tree.children[0]
 264     expect(firstChild.comment.text).to.equal('comment 1-1')
 265     expect(firstChild.children).to.have.lengthOf(1)
 266
 267     const childOfFirstChild = firstChild.children[0]
 268     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 269     expect(childOfFirstChild.children).to.have.lengthOf(1)
 270
 271     const childOfChildFirstChild = childOfFirstChild.children[0]
 272     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 273     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 274
 275     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 276     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 277     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 278   })
 279
 280   it('Should upload many videos on server 1', async function () {
 281     this.timeout(120000)
 282
 283     for (let i = 0; i < 10; i++) {
 284       const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid
 285       videoIdsServer1.push(uuid)
 286     }
 287
 288     await waitJobs(servers)
 289
 290     for (const id of videoIdsServer1) {
 291       await servers[1].videos.get({ id })
 292     }
 293
 294     await waitJobs(servers)
 295     await servers[1].sql.setActorFollowScores(20)
 296
 297     // Wait video expiration
 298     await wait(11000)
 299
 300     // Refresh video -> score + 10 = 30
 301     await servers[1].videos.get({ id: videoIdsServer1[0] })
 302
 303     await waitJobs(servers)
 304   })
 305
 306   it('Should remove followings that are down', async function () {
 307     this.timeout(120000)
 308
 309     await killallServers([ servers[0] ])
 310
 311     // Wait video expiration
 312     await wait(11000)
 313
 314     for (let i = 0; i < 5; i++) {
 315       try {
 316         await servers[1].videos.get({ id: videoIdsServer1[i] })
 317         await waitJobs([ servers[1] ])
 318         await wait(1500)
 319       } catch {}
 320     }
 321
 322     for (const id of videoIdsServer1) {
 323       await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 324     }
 325   })
 326
 327   after(async function () {
 328     await cleanupTests(servers)
 329   })
 330 })