5 files changed, 49 insertions, 27 deletions

diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index e12fc1dd4..7787186be 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -1,3 +1,4 @@
+import { InboxManager } from '@server/lib/activitypub/inbox-manager'
 import * as express from 'express'
 import { UserRight } from '../../../../shared/models/users'
 import { authenticate, ensureUserHasRight } from '../../../middlewares'
@@ -20,6 +21,7 @@ export {
 
 function getDebug (req: express.Request, res: express.Response) {
   return res.json({
-    ip: req.ip
-  }).end()
+    ip: req.ip,
+    activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting()
+  })
 }
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 5911d1a0f..fa0688a9e 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -307,7 +307,10 @@ async function removeUser (req: express.Request, res: express.Response) {
 
   auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
 
-  await user.destroy()
+  await sequelizeTypescript.transaction(async t => {
+    // Use a transaction to avoid inconsistencies with hooks (account/channel deletion & federation)
+    await user.destroy({ transaction: t })
+  })
 
   Hooks.runAction('action:api.user.deleted', { user })
 
diff --git a/server/lib/activitypub/inbox-manager.ts b/server/lib/activitypub/inbox-manager.ts
index 6d9bf7cf0..18ae49532 100644
--- a/server/lib/activitypub/inbox-manager.ts
+++ b/server/lib/activitypub/inbox-manager.ts
@@ -35,7 +35,7 @@ class InboxManager {
     })
 
     setInterval(() => {
-      StatsManager.Instance.updateInboxStats(this.messagesProcessed, this.inboxQueue.length())
+      StatsManager.Instance.updateInboxStats(this.messagesProcessed, this.getActivityPubMessagesWaiting())
     }, SCHEDULER_INTERVALS_MS.updateInboxStats)
   }
 
@@ -44,6 +44,10 @@ class InboxManager {
       .catch(err => logger.error('Cannot add options in inbox queue.', { options, err }))
   }
 
+  getActivityPubMessagesWaiting () {
+    return this.inboxQueue.length() + this.inboxQueue.running()
+  }
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 757e11845..6dd6fb44e 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -4,6 +4,7 @@ import 'mocha'
 import * as chai from 'chai'
 import { FfmpegCommand } from 'fluent-ffmpeg'
 import { LiveVideoCreate, VideoDetails, VideoPrivacy, VideoState } from '@shared/models'
+import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 import {
   addVideoToBlacklist,
   checkLiveCleanup,
@@ -23,9 +24,9 @@ import {
   updateCustomSubConfig,
   updateVideo,
   waitJobs,
+  waitUntilLiveEnded,
   waitUntilLivePublished
 } from '../../../../shared/extra-utils'
-import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 
 const expect = chai.expect
 
@@ -74,6 +75,12 @@ describe('Save replay setting', function () {
     }
   }
 
+  async function waitUntilLivePublishedOnAllServers (videoId: string) {
+    for (const server of servers) {
+      await waitUntilLivePublished(server.url, server.accessToken, videoId)
+    }
+  }
+
   before(async function () {
     this.timeout(120000)
 
@@ -125,10 +132,11 @@ describe('Save replay setting', function () {
     })
 
     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
-      this.timeout(20000)
+      this.timeout(30000)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
 
@@ -141,6 +149,9 @@ describe('Save replay setting', function () {
 
       await stopFfmpeg(ffmpegCommand)
 
+      for (const server of servers) {
+        await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID)
+      }
       await waitJobs(servers)
 
       // Live still exist, but cannot be played anymore
@@ -159,7 +170,8 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(false)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -185,7 +197,8 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(false)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -219,7 +232,7 @@ describe('Save replay setting', function () {
       this.timeout(20000)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
 
@@ -262,7 +275,7 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(true)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -288,7 +301,7 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(true)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index ff822f84d..19976ba8a 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -50,6 +50,12 @@ const expect = chai.expect
 describe('Test live', function () {
   let servers: ServerInfo[] = []
 
+  async function waitUntilLivePublishedOnAllServers (videoId: string) {
+    for (const server of servers) {
+      await waitUntilLivePublished(server.url, server.accessToken, videoId)
+    }
+  }
+
   before(async function () {
     this.timeout(120000)
 
@@ -390,7 +396,7 @@ describe('Test live', function () {
       liveVideoId = await createLiveWrapper(false)
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
 
       await testVideoResolutions(liveVideoId, [ 720 ])
@@ -406,7 +412,7 @@ describe('Test live', function () {
       liveVideoId = await createLiveWrapper(false)
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
 
       await testVideoResolutions(liveVideoId, resolutions)
@@ -423,7 +429,7 @@ describe('Test live', function () {
       liveVideoId = await createLiveWrapper(true)
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm')
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
 
       await testVideoResolutions(liveVideoId, resolutions)
@@ -433,7 +439,7 @@ describe('Test live', function () {
 
       await waitJobs(servers)
 
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
 
       const bitrateLimits = {
         720: 5000 * 1000, // 60FPS
@@ -514,7 +520,7 @@ describe('Test live', function () {
       liveVideoId = res.body.video.uuid
 
       command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
     })
 
@@ -602,10 +608,7 @@ describe('Test live', function () {
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 
-      for (const server of servers) {
-        await waitUntilLivePublished(server.url, server.accessToken, liveVideoUUID)
-      }
-
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
       await waitJobs(servers)
 
       for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
@@ -618,7 +621,6 @@ describe('Test live', function () {
       for (const server of servers) {
         await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID)
       }
-
       await waitJobs(servers)
 
       for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
@@ -654,10 +656,7 @@ describe('Test live', function () {
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 
-      for (const server of servers) {
-        await waitUntilLivePublished(server.url, server.accessToken, liveVideoUUID)
-      }
-
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
       await waitJobs(servers)
 
       expect(localLastVideoViews).to.equal(0)
@@ -691,7 +690,8 @@ describe('Test live', function () {
       socket.emit('subscribe', { videoId })
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
       await waitJobs(servers)
 
       expect(stateChanges).to.have.lengthOf(1)