11 files changed, 112 insertions, 52 deletions

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fe9851dc7..aba652586 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -42,7 +42,7 @@ jobs:
     env:
       PGUSER: peertube
       PGHOST: localhost
-      NODE_PENDING_JOB_WAIT: 2000
+      NODE_PENDING_JOB_WAIT: 500
 
     steps:
       - uses: actions/checkout@v2
@@ -93,7 +93,7 @@ jobs:
       - name: Run Test
         # external-plugins tests only run on schedule
         if: github.event_name == 'schedule' || matrix.test_suite != 'external-plugins'
-        run: NODE_PENDING_JOB_WAIT=2000 npm run ci -- ${{ matrix.test_suite }}
+        run: npm run ci -- ${{ matrix.test_suite }}
 
       - name: Display errors
         if: ${{ always() }}
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index e12fc1dd4..7787186be 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -1,3 +1,4 @@
+import { InboxManager } from '@server/lib/activitypub/inbox-manager'
 import * as express from 'express'
 import { UserRight } from '../../../../shared/models/users'
 import { authenticate, ensureUserHasRight } from '../../../middlewares'
@@ -20,6 +21,7 @@ export {
 
 function getDebug (req: express.Request, res: express.Response) {
   return res.json({
-    ip: req.ip
-  }).end()
+    ip: req.ip,
+    activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting()
+  })
 }
diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts
index 5911d1a0f..fa0688a9e 100644
--- a/server/controllers/api/users/index.ts
+++ b/server/controllers/api/users/index.ts
@@ -307,7 +307,10 @@ async function removeUser (req: express.Request, res: express.Response) {
 
   auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()))
 
-  await user.destroy()
+  await sequelizeTypescript.transaction(async t => {
+    // Use a transaction to avoid inconsistencies with hooks (account/channel deletion & federation)
+    await user.destroy({ transaction: t })
+  })
 
   Hooks.runAction('action:api.user.deleted', { user })
 
diff --git a/server/lib/activitypub/inbox-manager.ts b/server/lib/activitypub/inbox-manager.ts
index 6d9bf7cf0..18ae49532 100644
--- a/server/lib/activitypub/inbox-manager.ts
+++ b/server/lib/activitypub/inbox-manager.ts
@@ -35,7 +35,7 @@ class InboxManager {
     })
 
     setInterval(() => {
-      StatsManager.Instance.updateInboxStats(this.messagesProcessed, this.inboxQueue.length())
+      StatsManager.Instance.updateInboxStats(this.messagesProcessed, this.getActivityPubMessagesWaiting())
     }, SCHEDULER_INTERVALS_MS.updateInboxStats)
   }
 
@@ -44,6 +44,10 @@ class InboxManager {
       .catch(err => logger.error('Cannot add options in inbox queue.', { options, err }))
   }
 
+  getActivityPubMessagesWaiting () {
+    return this.inboxQueue.length() + this.inboxQueue.running()
+  }
+
   static get Instance () {
     return this.instance || (this.instance = new this())
   }
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 757e11845..6dd6fb44e 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -4,6 +4,7 @@ import 'mocha'
 import * as chai from 'chai'
 import { FfmpegCommand } from 'fluent-ffmpeg'
 import { LiveVideoCreate, VideoDetails, VideoPrivacy, VideoState } from '@shared/models'
+import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 import {
   addVideoToBlacklist,
   checkLiveCleanup,
@@ -23,9 +24,9 @@ import {
   updateCustomSubConfig,
   updateVideo,
   waitJobs,
+  waitUntilLiveEnded,
   waitUntilLivePublished
 } from '../../../../shared/extra-utils'
-import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 
 const expect = chai.expect
 
@@ -74,6 +75,12 @@ describe('Save replay setting', function () {
     }
   }
 
+  async function waitUntilLivePublishedOnAllServers (videoId: string) {
+    for (const server of servers) {
+      await waitUntilLivePublished(server.url, server.accessToken, videoId)
+    }
+  }
+
   before(async function () {
     this.timeout(120000)
 
@@ -125,10 +132,11 @@ describe('Save replay setting', function () {
     })
 
     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
-      this.timeout(20000)
+      this.timeout(30000)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
 
@@ -141,6 +149,9 @@ describe('Save replay setting', function () {
 
       await stopFfmpeg(ffmpegCommand)
 
+      for (const server of servers) {
+        await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID)
+      }
       await waitJobs(servers)
 
       // Live still exist, but cannot be played anymore
@@ -159,7 +170,8 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(false)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -185,7 +197,8 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(false)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -219,7 +232,7 @@ describe('Save replay setting', function () {
       this.timeout(20000)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
 
@@ -262,7 +275,7 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(true)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -288,7 +301,7 @@ describe('Save replay setting', function () {
       liveVideoUUID = await createLiveWrapper(true)
 
       ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
 
       await waitJobs(servers)
       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
index ff822f84d..19976ba8a 100644
--- a/server/tests/api/live/live.ts
+++ b/server/tests/api/live/live.ts
@@ -50,6 +50,12 @@ const expect = chai.expect
 describe('Test live', function () {
   let servers: ServerInfo[] = []
 
+  async function waitUntilLivePublishedOnAllServers (videoId: string) {
+    for (const server of servers) {
+      await waitUntilLivePublished(server.url, server.accessToken, videoId)
+    }
+  }
+
   before(async function () {
     this.timeout(120000)
 
@@ -390,7 +396,7 @@ describe('Test live', function () {
       liveVideoId = await createLiveWrapper(false)
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
 
       await testVideoResolutions(liveVideoId, [ 720 ])
@@ -406,7 +412,7 @@ describe('Test live', function () {
       liveVideoId = await createLiveWrapper(false)
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
 
       await testVideoResolutions(liveVideoId, resolutions)
@@ -423,7 +429,7 @@ describe('Test live', function () {
       liveVideoId = await createLiveWrapper(true)
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm')
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
 
       await testVideoResolutions(liveVideoId, resolutions)
@@ -433,7 +439,7 @@ describe('Test live', function () {
 
       await waitJobs(servers)
 
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
 
       const bitrateLimits = {
         720: 5000 * 1000, // 60FPS
@@ -514,7 +520,7 @@ describe('Test live', function () {
       liveVideoId = res.body.video.uuid
 
       command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId)
+      await waitUntilLivePublishedOnAllServers(liveVideoId)
       await waitJobs(servers)
     })
 
@@ -602,10 +608,7 @@ describe('Test live', function () {
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 
-      for (const server of servers) {
-        await waitUntilLivePublished(server.url, server.accessToken, liveVideoUUID)
-      }
-
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
       await waitJobs(servers)
 
       for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
@@ -618,7 +621,6 @@ describe('Test live', function () {
       for (const server of servers) {
         await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID)
       }
-
       await waitJobs(servers)
 
       for (const stateChanges of [ localStateChanges, remoteStateChanges ]) {
@@ -654,10 +656,7 @@ describe('Test live', function () {
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
 
-      for (const server of servers) {
-        await waitUntilLivePublished(server.url, server.accessToken, liveVideoUUID)
-      }
-
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
       await waitJobs(servers)
 
       expect(localLastVideoViews).to.equal(0)
@@ -691,7 +690,8 @@ describe('Test live', function () {
       socket.emit('subscribe', { videoId })
 
       const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID)
-      await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID)
+
+      await waitUntilLivePublishedOnAllServers(liveVideoUUID)
       await waitJobs(servers)
 
       expect(stateChanges).to.have.lengthOf(1)
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts
index 8f5c709ff..5c95a1b3e 100644
--- a/shared/extra-utils/index.ts
+++ b/shared/extra-utils/index.ts
@@ -1,32 +1,33 @@
-export * from './server/activitypub'
 export * from './bulk/bulk'
 export * from './cli/cli'
-export * from './server/clients'
-export * from './server/config'
-export * from './server/jobs'
-export * from './users/login'
+export * from './feeds/feeds'
+export * from './instances-index/mock-instances-index'
 export * from './miscs/miscs'
-export * from './miscs/stubs'
 export * from './miscs/sql'
-export * from './server/follows'
-export * from './requests/requests'
+export * from './miscs/stubs'
+export * from './moderation/abuses'
+export * from './plugins/mock-blocklist'
 export * from './requests/check-api-params'
-export * from './server/servers'
+export * from './requests/requests'
+export * from './search/videos'
+export * from './server/activitypub'
+export * from './server/clients'
+export * from './server/config'
+export * from './server/debug'
+export * from './server/follows'
+export * from './server/jobs'
 export * from './server/plugins'
-export * from './videos/video-playlists'
-export * from './users/users'
+export * from './server/servers'
 export * from './users/accounts'
-export * from './moderation/abuses'
-export * from './videos/services'
+export * from './users/login'
+export * from './users/users'
 export * from './videos/live'
+export * from './videos/services'
 export * from './videos/video-blacklist'
 export * from './videos/video-captions'
+export * from './videos/video-change-ownership'
 export * from './videos/video-channels'
 export * from './videos/video-comments'
+export * from './videos/video-playlists'
 export * from './videos/video-streaming-playlists'
 export * from './videos/videos'
-export * from './videos/video-change-ownership'
-export * from './feeds/feeds'
-export * from './instances-index/mock-instances-index'
-export * from './search/videos'
-export * from './plugins/mock-blocklist'
diff --git a/shared/extra-utils/server/debug.ts b/shared/extra-utils/server/debug.ts
new file mode 100644
index 000000000..5cf80a5fb
--- /dev/null
+++ b/shared/extra-utils/server/debug.ts
@@ -0,0 +1,19 @@
+import { makeGetRequest } from '../requests/requests'
+import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes'
+
+function getDebug (url: string, token: string) {
+  const path = '/api/v1/server/debug'
+
+  return makeGetRequest({
+    url,
+    path,
+    token,
+    statusCodeExpected: HttpStatusCode.OK_200
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getDebug
+}
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts
index a53749589..97971f960 100644
--- a/shared/extra-utils/server/jobs.ts
+++ b/shared/extra-utils/server/jobs.ts
@@ -1,7 +1,7 @@
 import * as request from 'supertest'
 import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
-import { makeGetRequest } from '../../../shared/extra-utils'
-import { Job, JobState, JobType } from '../../models'
+import { getDebug, makeGetRequest } from '../../../shared/extra-utils'
+import { Job, JobState, JobType, ServerDebug } from '../../models'
 import { wait } from '../miscs/miscs'
 import { ServerInfo } from './servers'
 
@@ -53,7 +53,10 @@ function getJobsListPaginationAndSort (options: {
 }
 
 async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
-  const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) : 2000
+  const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT
+    ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10)
+    : 500
+
   let servers: ServerInfo[]
 
   if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ]
@@ -75,16 +78,26 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
           start: 0,
           count: 10,
           sort: '-createdAt'
-        })
-          .then(res => res.body.data)
+        }).then(res => res.body.data)
           .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
           .then(jobs => {
             if (jobs.length !== 0) {
               pendingRequests = true
             }
           })
+
         tasks.push(p)
       }
+
+      const p = getDebug(server.url, server.accessToken)
+        .then(res => res.body)
+        .then((obj: ServerDebug) => {
+          if (obj.activityPubMessagesWaiting !== 0) {
+            pendingRequests = true
+          }
+        })
+
+      tasks.push(p)
     }
 
     return tasks
diff --git a/shared/models/server/index.ts b/shared/models/server/index.ts
index f253a3a79..b5163954a 100644
--- a/shared/models/server/index.ts
+++ b/shared/models/server/index.ts
@@ -7,5 +7,6 @@ export * from './emailer.model'
 export * from './job.model'
 export * from './log-level.type'
 export * from './server-config.model'
+export * from './server-debug.model'
 export * from './server-error-code.enum'
 export * from './server-stats.model'
diff --git a/shared/models/server/server-debug.model.ts b/shared/models/server/server-debug.model.ts
new file mode 100644
index 000000000..4b731bb90
--- /dev/null
+++ b/shared/models/server/server-debug.model.ts
@@ -0,0 +1,4 @@
+export interface ServerDebug {
+  ip: string
+  activityPubMessagesWaiting: number
+}