server/tests/api/server/handle-down.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { HttpStatusCode } from '@shared/core-utils'
   6 import {
   7   addVideoCommentReply,
   8   addVideoCommentThread,
   9   cleanupTests,
  10   closeAllSequelize,
  11   completeVideoCheck,
  12   flushAndRunMultipleServers,
  13   getJobsListPaginationAndSort,
  14   getVideo,
  15   getVideoCommentThreads,
  16   getVideosList,
  17   getVideoThreadComments,
  18   immutableAssign,
  19   killallServers,
  20   reRunServer,
  21   ServerInfo,
  22   setAccessTokensToServers,
  23   setActorFollowScores,
  24   updateVideo,
  25   uploadVideo,
  26   uploadVideoAndGetId,
  27   wait,
  28   waitJobs
  29 } from '@shared/extra-utils'
  30 import { JobState, Video, VideoCommentThreadTree, VideoPrivacy } from '@shared/models'
  31
  32 const expect = chai.expect
  33
  34 describe('Test handle downs', function () {
  35   let servers: ServerInfo[] = []
  36   let threadIdServer1: number
  37   let threadIdServer2: number
  38   let commentIdServer1: number
  39   let commentIdServer2: number
  40   let missedVideo1: Video
  41   let missedVideo2: Video
  42   let unlistedVideo: Video
  43
  44   const videoIdsServer1: string[] = []
  45
  46   const videoAttributes = {
  47     name: 'my super name for server 1',
  48     category: 5,
  49     licence: 4,
  50     language: 'ja',
  51     nsfw: true,
  52     privacy: VideoPrivacy.PUBLIC,
  53     description: 'my super description for server 1',
  54     support: 'my super support text for server 1',
  55     tags: [ 'tag1p1', 'tag2p1' ],
  56     fixture: 'video_short1.webm'
  57   }
  58
  59   const unlistedVideoAttributes = immutableAssign(videoAttributes, {
  60     privacy: VideoPrivacy.UNLISTED
  61   })
  62
  63   let checkAttributes: any
  64   let unlistedCheckAttributes: any
  65
  66   before(async function () {
  67     this.timeout(30000)
  68
  69     servers = await flushAndRunMultipleServers(3)
  70
  71     checkAttributes = {
  72       name: 'my super name for server 1',
  73       category: 5,
  74       licence: 4,
  75       language: 'ja',
  76       nsfw: true,
  77       description: 'my super description for server 1',
  78       support: 'my super support text for server 1',
  79       account: {
  80         name: 'root',
  81         host: 'localhost:' + servers[0].port
  82       },
  83       isLocal: false,
  84       duration: 10,
  85       tags: [ 'tag1p1', 'tag2p1' ],
  86       privacy: VideoPrivacy.PUBLIC,
  87       commentsEnabled: true,
  88       downloadEnabled: true,
  89       channel: {
  90         name: 'root_channel',
  91         displayName: 'Main root channel',
  92         description: '',
  93         isLocal: false
  94       },
  95       fixture: 'video_short1.webm',
  96       files: [
  97         {
  98           resolution: 720,
  99           size: 572456
 100         }
 101       ]
 102     }
 103     unlistedCheckAttributes = immutableAssign(checkAttributes, {
 104       privacy: VideoPrivacy.UNLISTED
 105     })
 106
 107     // Get the access tokens
 108     await setAccessTokensToServers(servers)
 109   })
 110
 111   it('Should remove followers that are often down', async function () {
 112     this.timeout(240000)
 113
 114     // Server 2 and 3 follow server 1
 115     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 116     await servers[2].followsCommand.follow({ targets: [ servers[0].url ] })
 117
 118     await waitJobs(servers)
 119
 120     // Upload a video to server 1
 121     await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 122
 123     await waitJobs(servers)
 124
 125     // And check all servers have this video
 126     for (const server of servers) {
 127       const res = await getVideosList(server.url)
 128       expect(res.body.data).to.be.an('array')
 129       expect(res.body.data).to.have.lengthOf(1)
 130     }
 131
 132     // Kill server 2
 133     killallServers([ servers[1] ])
 134
 135     // Remove server 2 follower
 136     for (let i = 0; i < 10; i++) {
 137       await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 138     }
 139
 140     await waitJobs([ servers[0], servers[2] ])
 141
 142     // Kill server 3
 143     killallServers([ servers[2] ])
 144
 145     const resLastVideo1 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 146     missedVideo1 = resLastVideo1.body.video
 147
 148     const resLastVideo2 = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
 149     missedVideo2 = resLastVideo2.body.video
 150
 151     // Unlisted video
 152     const resVideo = await uploadVideo(servers[0].url, servers[0].accessToken, unlistedVideoAttributes)
 153     unlistedVideo = resVideo.body.video
 154
 155     // Add comments to video 2
 156     {
 157       const text = 'thread 1'
 158       let resComment = await addVideoCommentThread(servers[0].url, servers[0].accessToken, missedVideo2.uuid, text)
 159       let comment = resComment.body.comment
 160       threadIdServer1 = comment.id
 161
 162       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-1')
 163       comment = resComment.body.comment
 164
 165       resComment = await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, comment.id, 'comment 1-2')
 166       commentIdServer1 = resComment.body.comment.id
 167     }
 168
 169     await waitJobs(servers[0])
 170     // Wait scheduler
 171     await wait(11000)
 172
 173     // Only server 3 is still a follower of server 1
 174     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 175     expect(body.data).to.be.an('array')
 176     expect(body.data).to.have.lengthOf(1)
 177     expect(body.data[0].follower.host).to.equal('localhost:' + servers[2].port)
 178   })
 179
 180   it('Should not have pending/processing jobs anymore', async function () {
 181     const states: JobState[] = [ 'waiting', 'active' ]
 182
 183     for (const state of states) {
 184       const res = await getJobsListPaginationAndSort({
 185         url: servers[0].url,
 186         accessToken: servers[0].accessToken,
 187         state: state,
 188         start: 0,
 189         count: 50,
 190         sort: '-createdAt'
 191       })
 192       expect(res.body.data).to.have.length(0)
 193     }
 194   })
 195
 196   it('Should re-follow server 1', async function () {
 197     this.timeout(35000)
 198
 199     await reRunServer(servers[1])
 200     await reRunServer(servers[2])
 201
 202     await servers[1].followsCommand.unfollow({ target: servers[0] })
 203     await waitJobs(servers)
 204
 205     await servers[1].followsCommand.follow({ targets: [ servers[0].url ] })
 206
 207     await waitJobs(servers)
 208
 209     const body = await servers[0].followsCommand.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
 210     expect(body.data).to.be.an('array')
 211     expect(body.data).to.have.lengthOf(2)
 212   })
 213
 214   it('Should send an update to server 3, and automatically fetch the video', async function () {
 215     this.timeout(15000)
 216
 217     const res1 = await getVideosList(servers[2].url)
 218     expect(res1.body.data).to.be.an('array')
 219     expect(res1.body.data).to.have.lengthOf(11)
 220
 221     await updateVideo(servers[0].url, servers[0].accessToken, missedVideo1.uuid, {})
 222     await updateVideo(servers[0].url, servers[0].accessToken, unlistedVideo.uuid, {})
 223
 224     await waitJobs(servers)
 225
 226     const res = await getVideosList(servers[2].url)
 227     expect(res.body.data).to.be.an('array')
 228     // 1 video is unlisted
 229     expect(res.body.data).to.have.lengthOf(12)
 230
 231     // Check unlisted video
 232     const resVideo = await getVideo(servers[2].url, unlistedVideo.uuid)
 233     expect(resVideo.body).not.to.be.undefined
 234
 235     await completeVideoCheck(servers[2].url, resVideo.body, unlistedCheckAttributes)
 236   })
 237
 238   it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
 239     this.timeout(25000)
 240
 241     await addVideoCommentReply(servers[0].url, servers[0].accessToken, missedVideo2.uuid, commentIdServer1, 'comment 1-3')
 242
 243     await waitJobs(servers)
 244
 245     const resVideo = await getVideo(servers[2].url, missedVideo2.uuid)
 246     expect(resVideo.body).not.to.be.undefined
 247
 248     {
 249       let resComment = await getVideoCommentThreads(servers[2].url, missedVideo2.uuid, 0, 5)
 250       expect(resComment.body.data).to.be.an('array')
 251       expect(resComment.body.data).to.have.lengthOf(1)
 252
 253       threadIdServer2 = resComment.body.data[0].id
 254
 255       resComment = await getVideoThreadComments(servers[2].url, missedVideo2.uuid, threadIdServer2)
 256
 257       const tree: VideoCommentThreadTree = resComment.body
 258       expect(tree.comment.text).equal('thread 1')
 259       expect(tree.children).to.have.lengthOf(1)
 260
 261       const firstChild = tree.children[0]
 262       expect(firstChild.comment.text).to.equal('comment 1-1')
 263       expect(firstChild.children).to.have.lengthOf(1)
 264
 265       const childOfFirstChild = firstChild.children[0]
 266       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 267       expect(childOfFirstChild.children).to.have.lengthOf(1)
 268
 269       const childOfChildFirstChild = childOfFirstChild.children[0]
 270       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 271       expect(childOfChildFirstChild.children).to.have.lengthOf(0)
 272
 273       commentIdServer2 = childOfChildFirstChild.comment.id
 274     }
 275   })
 276
 277   it('Should correctly reply to the comment', async function () {
 278     this.timeout(15000)
 279
 280     await addVideoCommentReply(servers[2].url, servers[2].accessToken, missedVideo2.uuid, commentIdServer2, 'comment 1-4')
 281
 282     await waitJobs(servers)
 283
 284     {
 285       const resComment = await getVideoThreadComments(servers[0].url, missedVideo2.uuid, threadIdServer1)
 286
 287       const tree: VideoCommentThreadTree = resComment.body
 288       expect(tree.comment.text).equal('thread 1')
 289       expect(tree.children).to.have.lengthOf(1)
 290
 291       const firstChild = tree.children[0]
 292       expect(firstChild.comment.text).to.equal('comment 1-1')
 293       expect(firstChild.children).to.have.lengthOf(1)
 294
 295       const childOfFirstChild = firstChild.children[0]
 296       expect(childOfFirstChild.comment.text).to.equal('comment 1-2')
 297       expect(childOfFirstChild.children).to.have.lengthOf(1)
 298
 299       const childOfChildFirstChild = childOfFirstChild.children[0]
 300       expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3')
 301       expect(childOfChildFirstChild.children).to.have.lengthOf(1)
 302
 303       const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0]
 304       expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4')
 305       expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0)
 306     }
 307   })
 308
 309   it('Should upload many videos on server 1', async function () {
 310     this.timeout(120000)
 311
 312     for (let i = 0; i < 10; i++) {
 313       const uuid = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video ' + i })).uuid
 314       videoIdsServer1.push(uuid)
 315     }
 316
 317     await waitJobs(servers)
 318
 319     for (const id of videoIdsServer1) {
 320       await getVideo(servers[1].url, id)
 321     }
 322
 323     await waitJobs(servers)
 324     await setActorFollowScores(servers[1].internalServerNumber, 20)
 325
 326     // Wait video expiration
 327     await wait(11000)
 328
 329     // Refresh video -> score + 10 = 30
 330     await getVideo(servers[1].url, videoIdsServer1[0])
 331
 332     await waitJobs(servers)
 333   })
 334
 335   it('Should remove followings that are down', async function () {
 336     this.timeout(120000)
 337
 338     killallServers([ servers[0] ])
 339
 340     // Wait video expiration
 341     await wait(11000)
 342
 343     for (let i = 0; i < 5; i++) {
 344       try {
 345         await getVideo(servers[1].url, videoIdsServer1[i])
 346         await waitJobs([ servers[1] ])
 347         await wait(1500)
 348       } catch {}
 349     }
 350
 351     for (const id of videoIdsServer1) {
 352       await getVideo(servers[1].url, id, HttpStatusCode.FORBIDDEN_403)
 353     }
 354   })
 355
 356   after(async function () {
 357     await closeAllSequelize([ servers[1] ])
 358
 359     await cleanupTests(servers)
 360   })
 361 })