server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import 'mocha'
   5 import { JobState, Video } from '../../../../shared/models'
   6 import { VideoPrivacy } from '../../../../shared/models/videos'
   7 import { VideoCommentThreadTree } from '../../../../shared/models/videos/video-comment.model'
   8
   9 import {
  10   cleanupTests,
  11   closeAllSequelize,
  12   completeVideoCheck,
  13   flushAndRunMultipleServers,
  14   getVideo,
  15   getVideosList,
  16   immutableAssign,
  17   killallServers,
  18   reRunServer,
  19   ServerInfo,
  20   setAccessTokensToServers,
  21   setActorFollowScores,
  22   unfollow,
  23   updateVideo,
  24   uploadVideo,
  25   uploadVideoAndGetId,
  26   wait
  27 } from '../../../../shared/extra-utils'
  28 import { follow, getFollowersListPaginationAndSort } from '../../../../shared/extra-utils/server/follows'
  29 import { getJobsListPaginationAndSort, waitJobs } from '../../../../shared/extra-utils/server/jobs'
  30 import {
  31   addVideoCommentReply,
  32   addVideoCommentThread,
  33   getVideoCommentThreads,
  34   getVideoThreadComments
  35 } from '../../../../shared/extra-utils/videos/video-comments'
  36
  37 const expect = chai.expect
  38
  39 describe('Test handle downs', function () {
  40   let servers: ServerInfo[] = []
  41   let threadIdServer1: number
  42   let threadIdServer2: number
  43   let commentIdServer1: number
  44   let commentIdServer2: number
  45   let missedVideo1: Video
  46   let missedVideo2: Video
  47   let unlistedVideo: Video
  48
  49   const videoIdsServer1: number[] = []
  50
  51   const videoAttributes = {
  52     name: 'my super name for server 1',
  53     category: 5,
  54     licence: 4,
  55     language: 'ja',
  56     nsfw: true,
  57     privacy: VideoPrivacy.PUBLIC,
  58     description: 'my super description for server 1',
  59     support: 'my super support text for server 1',
  60     tags: [ 'tag1p1', 'tag2p1' ],
  61     fixture: 'video_short1.webm'
  62   }
  63
  64   const unlistedVideoAttributes = immutableAssign(videoAttributes, {
  65     privacy: VideoPrivacy.UNLISTED
  66   })
  67
  68   let checkAttributes: any
  69   let unlistedCheckAttributes: any
  70
  71   before(async function () {
  72     this.timeout(30000)
  73
  74     servers = await flushAndRunMultipleServers(3)
  75
  76     checkAttributes = {
  77       name: 'my super name for server 1',
  78       category: 5,
  79       licence: 4,
  80       language: 'ja',
  81       nsfw: true,
  82       description: 'my super description for server 1',
  83       support: 'my super support text for server 1',
  84       account: {
  85         name: 'root',
  86         host: 'localhost:' + servers[0].port
  87       },
  88       isLocal: false,
  89       duration: 10,
  90       tags: [ 'tag1p1', 'tag2p1' ],
  91       privacy: VideoPrivacy.PUBLIC,
  92       commentsEnabled: true,
  93       downloadEnabled: true,
  94       channel: {
  95         name: 'root_channel',
  96         displayName: 'Main root channel',
  97         description: '',
  98         isLocal: false
  99       },
 100       fixture: 'video_short1.webm',
 101       files: [
 102         {
 103           resolution: 720,
 104           size: 572456
 105         }
 106       ]
 107     }
 108     unlistedCheckAttributes = immutableAssign(checkAttributes, {
 109       privacy: VideoPrivacy.UNLISTED
 110     })
 111
 112     // Get the access tokens
 113     await setAccessTokensToServers(servers)
 114   })
 115
 116   it('Should remove followers that are often down', async function () {
 117     this.timeout(240000)
 118
 119     // Server 2 and 3 follow server 1
 120     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
 121     await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken)
 122
 123     await waitJobs(servers)
 124
 125     // Upload a video to server 1
 126     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 127
 128     await waitJobs(servers)
 129
 130     // And check all servers have this video
 131     for (const server of servers) {
 132       const res = await getVideosList(server.url)
 133       expect(res.body.data).to.be.an('array')
 134       expect(res.body.data).to.have.lengthOf(1)
 135     }
 136
 137     // Kill server 2
 138     killallServers([ servers[1] ])
 139
 140     // Remove server 2 follower
 141     for (let i = 0; i < 10; i++) {
 142       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 143     }
 144
 145     await waitJobs(servers[0])
 146
 147     // Kill server 3
 148     killallServers([ servers[2] ])
 149
 150     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 151     missedVideo1 = resLastVideo1.body.video
 152
 153     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 154     missedVideo2 = resLastVideo2.body.video
 155
 156     // Unlisted video
 157     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 158     unlistedVideo = resVideo.body.video
 159
 160     // Add comments to video 2
 161     {
 162       const text = 'thread 1'
 163       let resComment = await addVideoCommentThread(servers[0].url, servers[0].accessToken, missedVideo2.uuid, text)
 164       let comment = resComment.body.comment
 165       threadIdServer1 = comment.id
 166
 167       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-1')
 168       comment = resComment.body.comment
 169
 170       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-2')
 171       commentIdServer1 = resComment.body.comment.id
 172     }
 173
 174     await waitJobs(servers[0])
 175     // Wait scheduler
 176     await wait(11000)
 177
 178     // Only server 3 is still a follower of server 1
 179     const res = await getFollowersListPaginationAndSort({ url: servers[0].url, start: 0, count: 2, sort: 'createdAt' })
 180     expect(res.body.data).to.be.an('array')
 181     expect(res.body.data).to.have.lengthOf(1)
 182     expect(res.body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 183   })
 184
 185   it('Should not have pending/processing jobs anymore', async function () {
 186     const states: JobState[] = [ 'waiting', 'active' ]
 187
 188     for (const state of states) {
 189       const res = await getJobsListPaginationAndSort({
 190         url: servers[0].url,
 191         accessToken: servers[0].accessToken,
 192         state: state,
 193         start: 0,
 194         count: 50,
 195         sort: '-createdAt'
 196       })
 197       expect(res.body.data).to.have.length(0)
 198     }
 199   })
 200
 201   it('Should re-follow server 1', async function () {
 202     this.timeout(35000)
 203
 204     await reRunServer(servers[1])
 205     await reRunServer(servers[2])
 206
 207     await unfollow(servers[1].url, servers[1].accessToken, servers[0])
 208     await waitJobs(servers)
 209
 210     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
 211
 212     await waitJobs(servers)
 213
 214     const res = await getFollowersListPaginationAndSort({ url: servers[0].url, start: 0, count: 2, sort: 'createdAt' })
 215     expect(res.body.data).to.be.an('array')
 216     expect(res.body.data).to.have.lengthOf(2)
 217   })
 218
 219   it('Should send an update to server 3, and automatically fetch the video', async function () {
 220     this.timeout(15000)
 221
 222     const res1 = await getVideosList(servers[2].url)
 223     expect(res1.body.data).to.be.an('array')
 224     expect(res1.body.data).to.have.lengthOf(11)
 225
 226     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 227     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 228
 229     await waitJobs(servers)
 230
 231     const res = await getVideosList(servers[2].url)
 232     expect(res.body.data).to.be.an('array')
 233     // 1 video is unlisted
 234     expect(res.body.data).to.have.lengthOf(12)
 235
 236     // Check unlisted video
 237     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 238     expect(resVideo.body).not.to.be.undefined
 239
 240     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 241   })
 242
 243   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 244     this.timeout(25000)
 245
 246     await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, commentIdServer1, 'comment 1-3')
 247
 248     await waitJobs(servers)
 249
 250     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 251     expect(resVideo.body).not.to.be.undefined
 252
 253     {
 254       let resComment = await getVideoCommentThreads(servers[2].url, missedVideo2.uuid, 0, 5)
 255       expect(resComment.body.data).to.be.an('array')
 256       expect(resComment.body.data).to.have.lengthOf(1)
 257
 258       threadIdServer2 = resComment.body.data[0].id
 259
 260       resComment = await getVideoThreadComments(servers[2].url, missedVideo2.uuid, threadIdServer2)
 261
 262       const tree: VideoCommentThreadTree = resComment.body
 263       expect(tree.comment.text).equal('thread 1')
 264       expect(tree.children).to.have.lengthOf(1)
 265
 266       const firstChild = tree.children[0]
 267       expect(firstChild.comment.text).to.equal('comment 1-1')
 268       expect(firstChild.children).to.have.lengthOf(1)
 269
 270       const childOfFirstChild = firstChild.children[0]
 271       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 272       expect(childOfFirstChild.children).to.have.lengthOf(1)
 273
 274       const childOfChildFirstChild = childOfFirstChild.children[0]
 275       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 276       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 277
 278       commentIdServer2 = childOfChildFirstChild.comment.id
 279     }
 280   })
 281
 282   it('Should correctly reply to the comment', async function () {
 283     this.timeout(15000)
 284
 285     await addVideoCommentReply(servers[2].url, servers[2].accessToken, missedVideo2.uuid, commentIdServer2, 'comment 1-4')
 286
 287     await waitJobs(servers)
 288
 289     {
 290       const resComment = await getVideoThreadComments(servers[0].url, missedVideo2.uuid, threadIdServer1)
 291
 292       const tree: VideoCommentThreadTree = resComment.body
 293       expect(tree.comment.text).equal('thread 1')
 294       expect(tree.children).to.have.lengthOf(1)
 295
 296       const firstChild = tree.children[0]
 297       expect(firstChild.comment.text).to.equal('comment 1-1')
 298       expect(firstChild.children).to.have.lengthOf(1)
 299
 300       const childOfFirstChild = firstChild.children[0]
 301       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 302       expect(childOfFirstChild.children).to.have.lengthOf(1)
 303
 304       const childOfChildFirstChild = childOfFirstChild.children[0]
 305       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 306       expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 307
 308       const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 309       expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 310       expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 311     }
 312   })
 313
 314   it('Should upload many videos on server 1', async function () {
 315     this.timeout(120000)
 316
 317     for (let i = 0; i < 10; i++) {
 318       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 319       videoIdsServer1.push(uuid)
 320     }
 321
 322     await waitJobs(servers)
 323
 324     for (const id of videoIdsServer1) {
 325       await getVideo(servers[1].url, id)
 326     }
 327
 328     await waitJobs(servers)
 329     await setActorFollowScores(servers[1].internalServerNumber, 20)
 330
 331     // Wait video expiration
 332     await wait(11000)
 333
 334     // Refresh video -> score + 10 = 30
 335     await getVideo(servers[1].url, videoIdsServer1[0])
 336
 337     await waitJobs(servers)
 338   })
 339
 340   it('Should remove followings that are down', async function () {
 341     this.timeout(120000)
 342
 343     killallServers([ servers[0] ])
 344
 345     // Wait video expiration
 346     await wait(11000)
 347
 348     for (let i = 0; i < 3; i++) {
 349       await getVideo(servers[1].url, videoIdsServer1[i])
 350       await wait(1000)
 351       await waitJobs([ servers[1] ])
 352     }
 353
 354     for (const id of videoIdsServer1) {
 355       await getVideo(servers[1].url, id, 403)
 356     }
 357   })
 358
 359   after(async function () {
 360     await closeAllSequelize([ servers[1] ])
 361
 362     await cleanupTests(servers)
 363   })
 364 })