server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { HttpStatusCode } from '@shared/core-utils'
   6 import {
   7   addVideoCommentReply,
   8   addVideoCommentThread,
   9   cleanupTests,
  10   closeAllSequelize,
  11   completeVideoCheck,
  12   flushAndRunMultipleServers,
  13   getVideo,
  14   getVideoCommentThreads,
  15   getVideosList,
  16   getVideoThreadComments,
  17   immutableAssign,
  18   killallServers,
  19   reRunServer,
  20   ServerInfo,
  21   setAccessTokensToServers,
  22   setActorFollowScores,
  23   updateVideo,
  24   uploadVideo,
  25   uploadVideoAndGetId,
  26   wait,
  27   waitJobs
  28 } from '@shared/extra-utils'
  29 import { JobState, Video, VideoCommentThreadTree, VideoPrivacy } from '@shared/models'
  30
  31 const expect = chai.expect
  32
  33 describe('Test handle downs', function () {
  34   let servers: ServerInfo[] = []
  35   let threadIdServer1: number
  36   let threadIdServer2: number
  37   let commentIdServer1: number
  38   let commentIdServer2: number
  39   let missedVideo1: Video
  40   let missedVideo2: Video
  41   let unlistedVideo: Video
  42
  43   const videoIdsServer1: string[] = []
  44
  45   const videoAttributes = {
  46     name: 'my super name for server 1',
  47     category: 5,
  48     licence: 4,
  49     language: 'ja',
  50     nsfw: true,
  51     privacy: VideoPrivacy.PUBLIC,
  52     description: 'my super description for server 1',
  53     support: 'my super support text for server 1',
  54     tags: [ 'tag1p1', 'tag2p1' ],
  55     fixture: 'video_short1.webm'
  56   }
  57
  58   const unlistedVideoAttributes = immutableAssign(videoAttributes, {
  59     privacy: VideoPrivacy.UNLISTED
  60   })
  61
  62   let checkAttributes: any
  63   let unlistedCheckAttributes: any
  64
  65   before(async function () {
  66     this.timeout(30000)
  67
  68     servers = await flushAndRunMultipleServers(3)
  69
  70     checkAttributes = {
  71       name: 'my super name for server 1',
  72       category: 5,
  73       licence: 4,
  74       language: 'ja',
  75       nsfw: true,
  76       description: 'my super description for server 1',
  77       support: 'my super support text for server 1',
  78       account: {
  79         name: 'root',
  80         host: 'localhost:' + servers[0].port
  81       },
  82       isLocal: false,
  83       duration: 10,
  84       tags: [ 'tag1p1', 'tag2p1' ],
  85       privacy: VideoPrivacy.PUBLIC,
  86       commentsEnabled: true,
  87       downloadEnabled: true,
  88       channel: {
  89         name: 'root_channel',
  90         displayName: 'Main root channel',
  91         description: '',
  92         isLocal: false
  93       },
  94       fixture: 'video_short1.webm',
  95       files: [
  96         {
  97           resolution: 720,
  98           size: 572456
  99         }
 100       ]
 101     }
 102     unlistedCheckAttributes = immutableAssign(checkAttributes, {
 103       privacy: VideoPrivacy.UNLISTED
 104     })
 105
 106     // Get the access tokens
 107     await setAccessTokensToServers(servers)
 108   })
 109
 110   it('Should remove followers that are often down', async function () {
 111     this.timeout(240000)
 112
 113     // Server 2 and 3 follow server 1
 114     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 115     await servers[2].followsCommand.follow({ targets: [ servers[0].url ] })
 116
 117     await waitJobs(servers)
 118
 119     // Upload a video to server 1
 120     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 121
 122     await waitJobs(servers)
 123
 124     // And check all servers have this video
 125     for (const server of servers) {
 126       const res = await getVideosList(server.url)
 127       expect(res.body.data).to.be.an('array')
 128       expect(res.body.data).to.have.lengthOf(1)
 129     }
 130
 131     // Kill server 2
 132     killallServers([ servers[1] ])
 133
 134     // Remove server 2 follower
 135     for (let i = 0; i < 10; i++) {
 136       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 137     }
 138
 139     await waitJobs([ servers[0], servers[2] ])
 140
 141     // Kill server 3
 142     killallServers([ servers[2] ])
 143
 144     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 145     missedVideo1 = resLastVideo1.body.video
 146
 147     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 148     missedVideo2 = resLastVideo2.body.video
 149
 150     // Unlisted video
 151     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 152     unlistedVideo = resVideo.body.video
 153
 154     // Add comments to video 2
 155     {
 156       const text = 'thread 1'
 157       let resComment = await addVideoCommentThread(servers[0].url, servers[0].accessToken, missedVideo2.uuid, text)
 158       let comment = resComment.body.comment
 159       threadIdServer1 = comment.id
 160
 161       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-1')
 162       comment = resComment.body.comment
 163
 164       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-2')
 165       commentIdServer1 = resComment.body.comment.id
 166     }
 167
 168     await waitJobs(servers[0])
 169     // Wait scheduler
 170     await wait(11000)
 171
 172     // Only server 3 is still a follower of server 1
 173     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 174     expect(body.data).to.be.an('array')
 175     expect(body.data).to.have.lengthOf(1)
 176     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 177   })
 178
 179   it('Should not have pending/processing jobs anymore', async function () {
 180     const states: JobState[] = [ 'waiting', 'active' ]
 181
 182     for (const state of states) {
 183       const body = await servers[0].jobsCommand.getJobsList({
 184         state: state,
 185         start: 0,
 186         count: 50,
 187         sort: '-createdAt'
 188       })
 189       expect(body.data).to.have.length(0)
 190     }
 191   })
 192
 193   it('Should re-follow server 1', async function () {
 194     this.timeout(35000)
 195
 196     await reRunServer(servers[1])
 197     await reRunServer(servers[2])
 198
 199     await servers[1].followsCommand.unfollow({ target: servers[0] })
 200     await waitJobs(servers)
 201
 202     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 203
 204     await waitJobs(servers)
 205
 206     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 207     expect(body.data).to.be.an('array')
 208     expect(body.data).to.have.lengthOf(2)
 209   })
 210
 211   it('Should send an update to server 3, and automatically fetch the video', async function () {
 212     this.timeout(15000)
 213
 214     const res1 = await getVideosList(servers[2].url)
 215     expect(res1.body.data).to.be.an('array')
 216     expect(res1.body.data).to.have.lengthOf(11)
 217
 218     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 219     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 220
 221     await waitJobs(servers)
 222
 223     const res = await getVideosList(servers[2].url)
 224     expect(res.body.data).to.be.an('array')
 225     // 1 video is unlisted
 226     expect(res.body.data).to.have.lengthOf(12)
 227
 228     // Check unlisted video
 229     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 230     expect(resVideo.body).not.to.be.undefined
 231
 232     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 233   })
 234
 235   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 236     this.timeout(25000)
 237
 238     await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, commentIdServer1, 'comment 1-3')
 239
 240     await waitJobs(servers)
 241
 242     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 243     expect(resVideo.body).not.to.be.undefined
 244
 245     {
 246       let resComment = await getVideoCommentThreads(servers[2].url, missedVideo2.uuid, 0, 5)
 247       expect(resComment.body.data).to.be.an('array')
 248       expect(resComment.body.data).to.have.lengthOf(1)
 249
 250       threadIdServer2 = resComment.body.data[0].id
 251
 252       resComment = await getVideoThreadComments(servers[2].url, missedVideo2.uuid, threadIdServer2)
 253
 254       const tree: VideoCommentThreadTree = resComment.body
 255       expect(tree.comment.text).equal('thread 1')
 256       expect(tree.children).to.have.lengthOf(1)
 257
 258       const firstChild = tree.children[0]
 259       expect(firstChild.comment.text).to.equal('comment 1-1')
 260       expect(firstChild.children).to.have.lengthOf(1)
 261
 262       const childOfFirstChild = firstChild.children[0]
 263       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 264       expect(childOfFirstChild.children).to.have.lengthOf(1)
 265
 266       const childOfChildFirstChild = childOfFirstChild.children[0]
 267       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 268       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 269
 270       commentIdServer2 = childOfChildFirstChild.comment.id
 271     }
 272   })
 273
 274   it('Should correctly reply to the comment', async function () {
 275     this.timeout(15000)
 276
 277     await addVideoCommentReply(servers[2].url, servers[2].accessToken, missedVideo2.uuid, commentIdServer2, 'comment 1-4')
 278
 279     await waitJobs(servers)
 280
 281     {
 282       const resComment = await getVideoThreadComments(servers[0].url, missedVideo2.uuid, threadIdServer1)
 283
 284       const tree: VideoCommentThreadTree = resComment.body
 285       expect(tree.comment.text).equal('thread 1')
 286       expect(tree.children).to.have.lengthOf(1)
 287
 288       const firstChild = tree.children[0]
 289       expect(firstChild.comment.text).to.equal('comment 1-1')
 290       expect(firstChild.children).to.have.lengthOf(1)
 291
 292       const childOfFirstChild = firstChild.children[0]
 293       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 294       expect(childOfFirstChild.children).to.have.lengthOf(1)
 295
 296       const childOfChildFirstChild = childOfFirstChild.children[0]
 297       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 298       expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 299
 300       const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 301       expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 302       expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 303     }
 304   })
 305
 306   it('Should upload many videos on server 1', async function () {
 307     this.timeout(120000)
 308
 309     for (let i = 0; i < 10; i++) {
 310       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 311       videoIdsServer1.push(uuid)
 312     }
 313
 314     await waitJobs(servers)
 315
 316     for (const id of videoIdsServer1) {
 317       await getVideo(servers[1].url, id)
 318     }
 319
 320     await waitJobs(servers)
 321     await setActorFollowScores(servers[1].internalServerNumber, 20)
 322
 323     // Wait video expiration
 324     await wait(11000)
 325
 326     // Refresh video -> score + 10 = 30
 327     await getVideo(servers[1].url, videoIdsServer1[0])
 328
 329     await waitJobs(servers)
 330   })
 331
 332   it('Should remove followings that are down', async function () {
 333     this.timeout(120000)
 334
 335     killallServers([ servers[0] ])
 336
 337     // Wait video expiration
 338     await wait(11000)
 339
 340     for (let i = 0; i < 5; i++) {
 341       try {
 342         await getVideo(servers[1].url, videoIdsServer1[i])
 343         await waitJobs([ servers[1] ])
 344         await wait(1500)
 345       } catch {}
 346     }
 347
 348     for (const id of videoIdsServer1) {
 349       await getVideo(servers[1].url, id, HttpStatusCode.FORBIDDEN_403)
 350     }
 351   })
 352
 353   after(async function () {
 354     await closeAllSequelize([ servers[1] ])
 355
 356     await cleanupTests(servers)
 357   })
 358 })