server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { JobState, Video } from '../../../../shared/models'
   6 import { VideoPrivacy } from '../../../../shared/models/videos'
   7 import { VideoCommentThreadTree } from '../../../../shared/models/videos/comment/video-comment.model'
   8
   9 import {
  10   cleanupTests,
  11   closeAllSequelize,
  12   completeVideoCheck,
  13   flushAndRunMultipleServers,
  14   getVideo,
  15   getVideosList,
  16   immutableAssign,
  17   killallServers,
  18   reRunServer,
  19   ServerInfo,
  20   setAccessTokensToServers,
  21   setActorFollowScores,
  22   unfollow,
  23   updateVideo,
  24   uploadVideo,
  25   uploadVideoAndGetId,
  26   wait
  27 } from '../../../../shared/extra-utils'
  28 import { follow, getFollowersListPaginationAndSort } from '../../../../shared/extra-utils/server/follows'
  29 import { getJobsListPaginationAndSort, waitJobs } from '../../../../shared/extra-utils/server/jobs'
  30 import {
  31   addVideoCommentReply,
  32   addVideoCommentThread,
  33   getVideoCommentThreads,
  34   getVideoThreadComments
  35 } from '../../../../shared/extra-utils/videos/video-comments'
  36 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
  37
  38 const expect = chai.expect
  39
  40 describe('Test handle downs', function () {
  41   let servers: ServerInfo[] = []
  42   let threadIdServer1: number
  43   let threadIdServer2: number
  44   let commentIdServer1: number
  45   let commentIdServer2: number
  46   let missedVideo1: Video
  47   let missedVideo2: Video
  48   let unlistedVideo: Video
  49
  50   const videoIdsServer1: string[] = []
  51
  52   const videoAttributes = {
  53     name: 'my super name for server 1',
  54     category: 5,
  55     licence: 4,
  56     language: 'ja',
  57     nsfw: true,
  58     privacy: VideoPrivacy.PUBLIC,
  59     description: 'my super description for server 1',
  60     support: 'my super support text for server 1',
  61     tags: [ 'tag1p1', 'tag2p1' ],
  62     fixture: 'video_short1.webm'
  63   }
  64
  65   const unlistedVideoAttributes = immutableAssign(videoAttributes, {
  66     privacy: VideoPrivacy.UNLISTED
  67   })
  68
  69   let checkAttributes: any
  70   let unlistedCheckAttributes: any
  71
  72   before(async function () {
  73     this.timeout(30000)
  74
  75     servers = await flushAndRunMultipleServers(3)
  76
  77     checkAttributes = {
  78       name: 'my super name for server 1',
  79       category: 5,
  80       licence: 4,
  81       language: 'ja',
  82       nsfw: true,
  83       description: 'my super description for server 1',
  84       support: 'my super support text for server 1',
  85       account: {
  86         name: 'root',
  87         host: 'localhost:' + servers[0].port
  88       },
  89       isLocal: false,
  90       duration: 10,
  91       tags: [ 'tag1p1', 'tag2p1' ],
  92       privacy: VideoPrivacy.PUBLIC,
  93       commentsEnabled: true,
  94       downloadEnabled: true,
  95       channel: {
  96         name: 'root_channel',
  97         displayName: 'Main root channel',
  98         description: '',
  99         isLocal: false
 100       },
 101       fixture: 'video_short1.webm',
 102       files: [
 103         {
 104           resolution: 720,
 105           size: 572456
 106         }
 107       ]
 108     }
 109     unlistedCheckAttributes = immutableAssign(checkAttributes, {
 110       privacy: VideoPrivacy.UNLISTED
 111     })
 112
 113     // Get the access tokens
 114     await setAccessTokensToServers(servers)
 115   })
 116
 117   it('Should remove followers that are often down', async function () {
 118     this.timeout(240000)
 119
 120     // Server 2 and 3 follow server 1
 121     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
 122     await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken)
 123
 124     await waitJobs(servers)
 125
 126     // Upload a video to server 1
 127     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 128
 129     await waitJobs(servers)
 130
 131     // And check all servers have this video
 132     for (const server of servers) {
 133       const res = await getVideosList(server.url)
 134       expect(res.body.data).to.be.an('array')
 135       expect(res.body.data).to.have.lengthOf(1)
 136     }
 137
 138     // Kill server 2
 139     killallServers([ servers[1] ])
 140
 141     // Remove server 2 follower
 142     for (let i = 0; i < 10; i++) {
 143       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 144     }
 145
 146     await waitJobs([ servers[0], servers[2] ])
 147
 148     // Kill server 3
 149     killallServers([ servers[2] ])
 150
 151     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 152     missedVideo1 = resLastVideo1.body.video
 153
 154     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 155     missedVideo2 = resLastVideo2.body.video
 156
 157     // Unlisted video
 158     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 159     unlistedVideo = resVideo.body.video
 160
 161     // Add comments to video 2
 162     {
 163       const text = 'thread 1'
 164       let resComment = await addVideoCommentThread(servers[0].url, servers[0].accessToken, missedVideo2.uuid, text)
 165       let comment = resComment.body.comment
 166       threadIdServer1 = comment.id
 167
 168       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-1')
 169       comment = resComment.body.comment
 170
 171       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-2')
 172       commentIdServer1 = resComment.body.comment.id
 173     }
 174
 175     await waitJobs(servers[0])
 176     // Wait scheduler
 177     await wait(11000)
 178
 179     // Only server 3 is still a follower of server 1
 180     const res = await getFollowersListPaginationAndSort({ url: servers[0].url, start: 0, count: 2, sort: 'createdAt' })
 181     expect(res.body.data).to.be.an('array')
 182     expect(res.body.data).to.have.lengthOf(1)
 183     expect(res.body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 184   })
 185
 186   it('Should not have pending/processing jobs anymore', async function () {
 187     const states: JobState[] = [ 'waiting', 'active' ]
 188
 189     for (const state of states) {
 190       const res = await getJobsListPaginationAndSort({
 191         url: servers[0].url,
 192         accessToken: servers[0].accessToken,
 193         state: state,
 194         start: 0,
 195         count: 50,
 196         sort: '-createdAt'
 197       })
 198       expect(res.body.data).to.have.length(0)
 199     }
 200   })
 201
 202   it('Should re-follow server 1', async function () {
 203     this.timeout(35000)
 204
 205     await reRunServer(servers[1])
 206     await reRunServer(servers[2])
 207
 208     await unfollow(servers[1].url, servers[1].accessToken, servers[0])
 209     await waitJobs(servers)
 210
 211     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
 212
 213     await waitJobs(servers)
 214
 215     const res = await getFollowersListPaginationAndSort({ url: servers[0].url, start: 0, count: 2, sort: 'createdAt' })
 216     expect(res.body.data).to.be.an('array')
 217     expect(res.body.data).to.have.lengthOf(2)
 218   })
 219
 220   it('Should send an update to server 3, and automatically fetch the video', async function () {
 221     this.timeout(15000)
 222
 223     const res1 = await getVideosList(servers[2].url)
 224     expect(res1.body.data).to.be.an('array')
 225     expect(res1.body.data).to.have.lengthOf(11)
 226
 227     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 228     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 229
 230     await waitJobs(servers)
 231
 232     const res = await getVideosList(servers[2].url)
 233     expect(res.body.data).to.be.an('array')
 234     // 1 video is unlisted
 235     expect(res.body.data).to.have.lengthOf(12)
 236
 237     // Check unlisted video
 238     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 239     expect(resVideo.body).not.to.be.undefined
 240
 241     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 242   })
 243
 244   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 245     this.timeout(25000)
 246
 247     await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, commentIdServer1, 'comment 1-3')
 248
 249     await waitJobs(servers)
 250
 251     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 252     expect(resVideo.body).not.to.be.undefined
 253
 254     {
 255       let resComment = await getVideoCommentThreads(servers[2].url, missedVideo2.uuid, 0, 5)
 256       expect(resComment.body.data).to.be.an('array')
 257       expect(resComment.body.data).to.have.lengthOf(1)
 258
 259       threadIdServer2 = resComment.body.data[0].id
 260
 261       resComment = await getVideoThreadComments(servers[2].url, missedVideo2.uuid, threadIdServer2)
 262
 263       const tree: VideoCommentThreadTree = resComment.body
 264       expect(tree.comment.text).equal('thread 1')
 265       expect(tree.children).to.have.lengthOf(1)
 266
 267       const firstChild = tree.children[0]
 268       expect(firstChild.comment.text).to.equal('comment 1-1')
 269       expect(firstChild.children).to.have.lengthOf(1)
 270
 271       const childOfFirstChild = firstChild.children[0]
 272       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 273       expect(childOfFirstChild.children).to.have.lengthOf(1)
 274
 275       const childOfChildFirstChild = childOfFirstChild.children[0]
 276       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 277       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 278
 279       commentIdServer2 = childOfChildFirstChild.comment.id
 280     }
 281   })
 282
 283   it('Should correctly reply to the comment', async function () {
 284     this.timeout(15000)
 285
 286     await addVideoCommentReply(servers[2].url, servers[2].accessToken, missedVideo2.uuid, commentIdServer2, 'comment 1-4')
 287
 288     await waitJobs(servers)
 289
 290     {
 291       const resComment = await getVideoThreadComments(servers[0].url, missedVideo2.uuid, threadIdServer1)
 292
 293       const tree: VideoCommentThreadTree = resComment.body
 294       expect(tree.comment.text).equal('thread 1')
 295       expect(tree.children).to.have.lengthOf(1)
 296
 297       const firstChild = tree.children[0]
 298       expect(firstChild.comment.text).to.equal('comment 1-1')
 299       expect(firstChild.children).to.have.lengthOf(1)
 300
 301       const childOfFirstChild = firstChild.children[0]
 302       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 303       expect(childOfFirstChild.children).to.have.lengthOf(1)
 304
 305       const childOfChildFirstChild = childOfFirstChild.children[0]
 306       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 307       expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 308
 309       const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 310       expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 311       expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 312     }
 313   })
 314
 315   it('Should upload many videos on server 1', async function () {
 316     this.timeout(120000)
 317
 318     for (let i = 0; i < 10; i++) {
 319       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 320       videoIdsServer1.push(uuid)
 321     }
 322
 323     await waitJobs(servers)
 324
 325     for (const id of videoIdsServer1) {
 326       await getVideo(servers[1].url, id)
 327     }
 328
 329     await waitJobs(servers)
 330     await setActorFollowScores(servers[1].internalServerNumber, 20)
 331
 332     // Wait video expiration
 333     await wait(11000)
 334
 335     // Refresh video -> score + 10 = 30
 336     await getVideo(servers[1].url, videoIdsServer1[0])
 337
 338     await waitJobs(servers)
 339   })
 340
 341   it('Should remove followings that are down', async function () {
 342     this.timeout(120000)
 343
 344     killallServers([ servers[0] ])
 345
 346     // Wait video expiration
 347     await wait(11000)
 348
 349     for (let i = 0; i < 5; i++) {
 350       try {
 351         await getVideo(servers[1].url, videoIdsServer1[i])
 352         await waitJobs([ servers[1] ])
 353         await wait(1500)
 354       } catch {}
 355     }
 356
 357     for (const id of videoIdsServer1) {
 358       await getVideo(servers[1].url, id, HttpStatusCode.FORBIDDEN_403)
 359     }
 360   })
 361
 362   after(async function () {
 363     await closeAllSequelize([ servers[1] ])
 364
 365     await cleanupTests(servers)
 366   })
 367 })