server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { HttpStatusCode } from '@shared/core-utils'
   6 import {
   7   cleanupTests,
   8   CommentsCommand,
   9   completeVideoCheck,
  10   flushAndRunMultipleServers,
  11   getVideo,
  12   getVideosList,
  13   killallServers,
  14   reRunServer,
  15   ServerInfo,
  16   setAccessTokensToServers,
  17   updateVideo,
  18   uploadVideo,
  19   uploadVideoAndGetId,
  20   wait,
  21   waitJobs
  22 } from '@shared/extra-utils'
  23 import { JobState, Video, VideoPrivacy } from '@shared/models'
  24
  25 const expect = chai.expect
  26
  27 describe('Test handle downs', function () {
  28   let servers: ServerInfo[] = []
  29   let threadIdServer1: number
  30   let threadIdServer2: number
  31   let commentIdServer1: number
  32   let commentIdServer2: number
  33   let missedVideo1: Video
  34   let missedVideo2: Video
  35   let unlistedVideo: Video
  36
  37   const videoIdsServer1: string[] = []
  38
  39   const videoAttributes = {
  40     name: 'my super name for server 1',
  41     category: 5,
  42     licence: 4,
  43     language: 'ja',
  44     nsfw: true,
  45     privacy: VideoPrivacy.PUBLIC,
  46     description: 'my super description for server 1',
  47     support: 'my super support text for server 1',
  48     tags: [ 'tag1p1', 'tag2p1' ],
  49     fixture: 'video_short1.webm'
  50   }
  51
  52   const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED }
  53
  54   let checkAttributes: any
  55   let unlistedCheckAttributes: any
  56
  57   let commentCommands: CommentsCommand[]
  58
  59   before(async function () {
  60     this.timeout(30000)
  61
  62     servers = await flushAndRunMultipleServers(3)
  63     commentCommands = servers.map(s => s.commentsCommand)
  64
  65     checkAttributes = {
  66       name: 'my super name for server 1',
  67       category: 5,
  68       licence: 4,
  69       language: 'ja',
  70       nsfw: true,
  71       description: 'my super description for server 1',
  72       support: 'my super support text for server 1',
  73       account: {
  74         name: 'root',
  75         host: 'localhost:' + servers[0].port
  76       },
  77       isLocal: false,
  78       duration: 10,
  79       tags: [ 'tag1p1', 'tag2p1' ],
  80       privacy: VideoPrivacy.PUBLIC,
  81       commentsEnabled: true,
  82       downloadEnabled: true,
  83       channel: {
  84         name: 'root_channel',
  85         displayName: 'Main root channel',
  86         description: '',
  87         isLocal: false
  88       },
  89       fixture: 'video_short1.webm',
  90       files: [
  91         {
  92           resolution: 720,
  93           size: 572456
  94         }
  95       ]
  96     }
  97     unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED }
  98
  99     // Get the access tokens
 100     await setAccessTokensToServers(servers)
 101   })
 102
 103   it('Should remove followers that are often down', async function () {
 104     this.timeout(240000)
 105
 106     // Server 2 and 3 follow server 1
 107     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 108     await servers[2].followsCommand.follow({ targets: [ servers[0].url ] })
 109
 110     await waitJobs(servers)
 111
 112     // Upload a video to server 1
 113     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 114
 115     await waitJobs(servers)
 116
 117     // And check all servers have this video
 118     for (const server of servers) {
 119       const res = await getVideosList(server.url)
 120       expect(res.body.data).to.be.an('array')
 121       expect(res.body.data).to.have.lengthOf(1)
 122     }
 123
 124     // Kill server 2
 125     await killallServers([ servers[1] ])
 126
 127     // Remove server 2 follower
 128     for (let i = 0; i < 10; i++) {
 129       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 130     }
 131
 132     await waitJobs([ servers[0], servers[2] ])
 133
 134     // Kill server 3
 135     await killallServers([ servers[2] ])
 136
 137     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 138     missedVideo1 = resLastVideo1.body.video
 139
 140     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 141     missedVideo2 = resLastVideo2.body.video
 142
 143     // Unlisted video
 144     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 145     unlistedVideo = resVideo.body.video
 146
 147     // Add comments to video 2
 148     {
 149       const text = 'thread 1'
 150       let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text })
 151       threadIdServer1 = comment.id
 152
 153       comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' })
 154
 155       const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' })
 156       commentIdServer1 = created.id
 157     }
 158
 159     await waitJobs(servers[0])
 160     // Wait scheduler
 161     await wait(11000)
 162
 163     // Only server 3 is still a follower of server 1
 164     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 165     expect(body.data).to.be.an('array')
 166     expect(body.data).to.have.lengthOf(1)
 167     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 168   })
 169
 170   it('Should not have pending/processing jobs anymore', async function () {
 171     const states: JobState[] = [ 'waiting', 'active' ]
 172
 173     for (const state of states) {
 174       const body = await servers[0].jobsCommand.getJobsList({
 175         state: state,
 176         start: 0,
 177         count: 50,
 178         sort: '-createdAt'
 179       })
 180       expect(body.data).to.have.length(0)
 181     }
 182   })
 183
 184   it('Should re-follow server 1', async function () {
 185     this.timeout(35000)
 186
 187     await reRunServer(servers[1])
 188     await reRunServer(servers[2])
 189
 190     await servers[1].followsCommand.unfollow({ target: servers[0] })
 191     await waitJobs(servers)
 192
 193     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 194
 195     await waitJobs(servers)
 196
 197     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 198     expect(body.data).to.be.an('array')
 199     expect(body.data).to.have.lengthOf(2)
 200   })
 201
 202   it('Should send an update to server 3, and automatically fetch the video', async function () {
 203     this.timeout(15000)
 204
 205     const res1 = await getVideosList(servers[2].url)
 206     expect(res1.body.data).to.be.an('array')
 207     expect(res1.body.data).to.have.lengthOf(11)
 208
 209     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 210     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 211
 212     await waitJobs(servers)
 213
 214     const res = await getVideosList(servers[2].url)
 215     expect(res.body.data).to.be.an('array')
 216     // 1 video is unlisted
 217     expect(res.body.data).to.have.lengthOf(12)
 218
 219     // Check unlisted video
 220     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 221     expect(resVideo.body).not.to.be.undefined
 222
 223     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 224   })
 225
 226   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 227     this.timeout(25000)
 228
 229     await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
 230
 231     await waitJobs(servers)
 232
 233     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 234     expect(resVideo.body).not.to.be.undefined
 235
 236     {
 237       const { data } = await servers[2].commentsCommand.listThreads({ videoId: missedVideo2.uuid })
 238       expect(data).to.be.an('array')
 239       expect(data).to.have.lengthOf(1)
 240
 241       threadIdServer2 = data[0].id
 242
 243       const tree = await servers[2].commentsCommand.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 })
 244       expect(tree.comment.text).equal('thread 1')
 245       expect(tree.children).to.have.lengthOf(1)
 246
 247       const firstChild = tree.children[0]
 248       expect(firstChild.comment.text).to.equal('comment 1-1')
 249       expect(firstChild.children).to.have.lengthOf(1)
 250
 251       const childOfFirstChild = firstChild.children[0]
 252       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 253       expect(childOfFirstChild.children).to.have.lengthOf(1)
 254
 255       const childOfChildFirstChild = childOfFirstChild.children[0]
 256       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 257       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 258
 259       commentIdServer2 = childOfChildFirstChild.comment.id
 260     }
 261   })
 262
 263   it('Should correctly reply to the comment', async function () {
 264     this.timeout(15000)
 265
 266     await servers[2].commentsCommand.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' })
 267
 268     await waitJobs(servers)
 269
 270     const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 })
 271
 272     expect(tree.comment.text).equal('thread 1')
 273     expect(tree.children).to.have.lengthOf(1)
 274
 275     const firstChild = tree.children[0]
 276     expect(firstChild.comment.text).to.equal('comment 1-1')
 277     expect(firstChild.children).to.have.lengthOf(1)
 278
 279     const childOfFirstChild = firstChild.children[0]
 280     expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 281     expect(childOfFirstChild.children).to.have.lengthOf(1)
 282
 283     const childOfChildFirstChild = childOfFirstChild.children[0]
 284     expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 285     expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 286
 287     const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 288     expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 289     expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 290   })
 291
 292   it('Should upload many videos on server 1', async function () {
 293     this.timeout(120000)
 294
 295     for (let i = 0; i < 10; i++) {
 296       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 297       videoIdsServer1.push(uuid)
 298     }
 299
 300     await waitJobs(servers)
 301
 302     for (const id of videoIdsServer1) {
 303       await getVideo(servers[1].url, id)
 304     }
 305
 306     await waitJobs(servers)
 307     await servers[1].sqlCommand.setActorFollowScores(20)
 308
 309     // Wait video expiration
 310     await wait(11000)
 311
 312     // Refresh video -> score + 10 = 30
 313     await getVideo(servers[1].url, videoIdsServer1[0])
 314
 315     await waitJobs(servers)
 316   })
 317
 318   it('Should remove followings that are down', async function () {
 319     this.timeout(120000)
 320
 321     await killallServers([ servers[0] ])
 322
 323     // Wait video expiration
 324     await wait(11000)
 325
 326     for (let i = 0; i < 5; i++) {
 327       try {
 328         await getVideo(servers[1].url, videoIdsServer1[i])
 329         await waitJobs([ servers[1] ])
 330         await wait(1500)
 331       } catch {}
 332     }
 333
 334     for (const id of videoIdsServer1) {
 335       await getVideo(servers[1].url, id, HttpStatusCode.FORBIDDEN_403)
 336     }
 337   })
 338
 339   after(async function () {
 340     await cleanupTests(servers)
 341   })
 342 })