From fae6e4da8f516a9d6c3bad9bf6f35811ccacbad8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 18 Feb 2021 14:44:12 +0100 Subject: [PATCH] Try to speed up server tests --- .github/workflows/test.yml | 4 +-- server/controllers/api/server/debug.ts | 6 ++-- server/controllers/api/users/index.ts | 5 ++- server/lib/activitypub/inbox-manager.ts | 6 +++- server/tests/api/live/live-save-replay.ts | 29 ++++++++++++----- server/tests/api/live/live.ts | 30 +++++++++--------- shared/extra-utils/index.ts | 37 +++++++++++----------- shared/extra-utils/server/debug.ts | 19 +++++++++++ shared/extra-utils/server/jobs.ts | 23 +++++++++++--- shared/models/server/index.ts | 1 + shared/models/server/server-debug.model.ts | 4 +++ 11 files changed, 112 insertions(+), 52 deletions(-) create mode 100644 shared/extra-utils/server/debug.ts create mode 100644 shared/models/server/server-debug.model.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe9851dc7..aba652586 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: env: PGUSER: peertube PGHOST: localhost - NODE_PENDING_JOB_WAIT: 2000 + NODE_PENDING_JOB_WAIT: 500 steps: - uses: actions/checkout@v2 @@ -93,7 +93,7 @@ jobs: - name: Run Test # external-plugins tests only run on schedule if: github.event_name == 'schedule' || matrix.test_suite != 'external-plugins' - run: NODE_PENDING_JOB_WAIT=2000 npm run ci -- ${{ matrix.test_suite }} + run: npm run ci -- ${{ matrix.test_suite }} - name: Display errors if: ${{ always() }} diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index e12fc1dd4..7787186be 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts @@ -1,3 +1,4 @@ +import { InboxManager } from '@server/lib/activitypub/inbox-manager' import * as express from 'express' import { UserRight } from '../../../../shared/models/users' import { authenticate, ensureUserHasRight } from '../../../middlewares' @@ -20,6 +21,7 @@ export { function getDebug (req: express.Request, res: express.Response) { return res.json({ - ip: req.ip - }).end() + ip: req.ip, + activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting() + }) } diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts index 5911d1a0f..fa0688a9e 100644 --- a/server/controllers/api/users/index.ts +++ b/server/controllers/api/users/index.ts @@ -307,7 +307,10 @@ async function removeUser (req: express.Request, res: express.Response) { auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) - await user.destroy() + await sequelizeTypescript.transaction(async t => { + // Use a transaction to avoid inconsistencies with hooks (account/channel deletion & federation) + await user.destroy({ transaction: t }) + }) Hooks.runAction('action:api.user.deleted', { user }) diff --git a/server/lib/activitypub/inbox-manager.ts b/server/lib/activitypub/inbox-manager.ts index 6d9bf7cf0..18ae49532 100644 --- a/server/lib/activitypub/inbox-manager.ts +++ b/server/lib/activitypub/inbox-manager.ts @@ -35,7 +35,7 @@ class InboxManager { }) setInterval(() => { - StatsManager.Instance.updateInboxStats(this.messagesProcessed, this.inboxQueue.length()) + StatsManager.Instance.updateInboxStats(this.messagesProcessed, this.getActivityPubMessagesWaiting()) }, SCHEDULER_INTERVALS_MS.updateInboxStats) } @@ -44,6 +44,10 @@ class InboxManager { .catch(err => logger.error('Cannot add options in inbox queue.', { options, err })) } + getActivityPubMessagesWaiting () { + return this.inboxQueue.length() + this.inboxQueue.running() + } + static get Instance () { return this.instance || (this.instance = new this()) } diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts index 757e11845..6dd6fb44e 100644 --- a/server/tests/api/live/live-save-replay.ts +++ b/server/tests/api/live/live-save-replay.ts @@ -4,6 +4,7 @@ import 'mocha' import * as chai from 'chai' import { FfmpegCommand } from 'fluent-ffmpeg' import { LiveVideoCreate, VideoDetails, VideoPrivacy, VideoState } from '@shared/models' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { addVideoToBlacklist, checkLiveCleanup, @@ -23,9 +24,9 @@ import { updateCustomSubConfig, updateVideo, waitJobs, + waitUntilLiveEnded, waitUntilLivePublished } from '../../../../shared/extra-utils' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' const expect = chai.expect @@ -74,6 +75,12 @@ describe('Save replay setting', function () { } } + async function waitUntilLivePublishedOnAllServers (videoId: string) { + for (const server of servers) { + await waitUntilLivePublished(server.url, server.accessToken, videoId) + } + } + before(async function () { this.timeout(120000) @@ -125,10 +132,11 @@ describe('Save replay setting', function () { }) it('Should correctly have updated the live and federated it when streaming in the live', async function () { - this.timeout(20000) + this.timeout(30000) ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) @@ -141,6 +149,9 @@ describe('Save replay setting', function () { await stopFfmpeg(ffmpegCommand) + for (const server of servers) { + await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID) + } await waitJobs(servers) // Live still exist, but cannot be played anymore @@ -159,7 +170,8 @@ describe('Save replay setting', function () { liveVideoUUID = await createLiveWrapper(false) ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) @@ -185,7 +197,8 @@ describe('Save replay setting', function () { liveVideoUUID = await createLiveWrapper(false) ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) @@ -219,7 +232,7 @@ describe('Save replay setting', function () { this.timeout(20000) ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) @@ -262,7 +275,7 @@ describe('Save replay setting', function () { liveVideoUUID = await createLiveWrapper(true) ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) @@ -288,7 +301,7 @@ describe('Save replay setting', function () { liveVideoUUID = await createLiveWrapper(true) ffmpegCommand = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index ff822f84d..19976ba8a 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts @@ -50,6 +50,12 @@ const expect = chai.expect describe('Test live', function () { let servers: ServerInfo[] = [] + async function waitUntilLivePublishedOnAllServers (videoId: string) { + for (const server of servers) { + await waitUntilLivePublished(server.url, server.accessToken, videoId) + } + } + before(async function () { this.timeout(120000) @@ -390,7 +396,7 @@ describe('Test live', function () { liveVideoId = await createLiveWrapper(false) const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId) + await waitUntilLivePublishedOnAllServers(liveVideoId) await waitJobs(servers) await testVideoResolutions(liveVideoId, [ 720 ]) @@ -406,7 +412,7 @@ describe('Test live', function () { liveVideoId = await createLiveWrapper(false) const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId) + await waitUntilLivePublishedOnAllServers(liveVideoId) await waitJobs(servers) await testVideoResolutions(liveVideoId, resolutions) @@ -423,7 +429,7 @@ describe('Test live', function () { liveVideoId = await createLiveWrapper(true) const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm') - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId) + await waitUntilLivePublishedOnAllServers(liveVideoId) await waitJobs(servers) await testVideoResolutions(liveVideoId, resolutions) @@ -433,7 +439,7 @@ describe('Test live', function () { await waitJobs(servers) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId) + await waitUntilLivePublishedOnAllServers(liveVideoId) const bitrateLimits = { 720: 5000 * 1000, // 60FPS @@ -514,7 +520,7 @@ describe('Test live', function () { liveVideoId = res.body.video.uuid command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoId) + await waitUntilLivePublishedOnAllServers(liveVideoId) await waitJobs(servers) }) @@ -602,10 +608,7 @@ describe('Test live', function () { const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - for (const server of servers) { - await waitUntilLivePublished(server.url, server.accessToken, liveVideoUUID) - } - + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) for (const stateChanges of [ localStateChanges, remoteStateChanges ]) { @@ -618,7 +621,6 @@ describe('Test live', function () { for (const server of servers) { await waitUntilLiveEnded(server.url, server.accessToken, liveVideoUUID) } - await waitJobs(servers) for (const stateChanges of [ localStateChanges, remoteStateChanges ]) { @@ -654,10 +656,7 @@ describe('Test live', function () { const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - for (const server of servers) { - await waitUntilLivePublished(server.url, server.accessToken, liveVideoUUID) - } - + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) expect(localLastVideoViews).to.equal(0) @@ -691,7 +690,8 @@ describe('Test live', function () { socket.emit('subscribe', { videoId }) const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoUUID) - await waitUntilLivePublished(servers[0].url, servers[0].accessToken, liveVideoUUID) + + await waitUntilLivePublishedOnAllServers(liveVideoUUID) await waitJobs(servers) expect(stateChanges).to.have.lengthOf(1) diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts index 8f5c709ff..5c95a1b3e 100644 --- a/shared/extra-utils/index.ts +++ b/shared/extra-utils/index.ts @@ -1,32 +1,33 @@ -export * from './server/activitypub' export * from './bulk/bulk' export * from './cli/cli' -export * from './server/clients' -export * from './server/config' -export * from './server/jobs' -export * from './users/login' +export * from './feeds/feeds' +export * from './instances-index/mock-instances-index' export * from './miscs/miscs' -export * from './miscs/stubs' export * from './miscs/sql' -export * from './server/follows' -export * from './requests/requests' +export * from './miscs/stubs' +export * from './moderation/abuses' +export * from './plugins/mock-blocklist' export * from './requests/check-api-params' -export * from './server/servers' +export * from './requests/requests' +export * from './search/videos' +export * from './server/activitypub' +export * from './server/clients' +export * from './server/config' +export * from './server/debug' +export * from './server/follows' +export * from './server/jobs' export * from './server/plugins' -export * from './videos/video-playlists' -export * from './users/users' +export * from './server/servers' export * from './users/accounts' -export * from './moderation/abuses' -export * from './videos/services' +export * from './users/login' +export * from './users/users' export * from './videos/live' +export * from './videos/services' export * from './videos/video-blacklist' export * from './videos/video-captions' +export * from './videos/video-change-ownership' export * from './videos/video-channels' export * from './videos/video-comments' +export * from './videos/video-playlists' export * from './videos/video-streaming-playlists' export * from './videos/videos' -export * from './videos/video-change-ownership' -export * from './feeds/feeds' -export * from './instances-index/mock-instances-index' -export * from './search/videos' -export * from './plugins/mock-blocklist' diff --git a/shared/extra-utils/server/debug.ts b/shared/extra-utils/server/debug.ts new file mode 100644 index 000000000..5cf80a5fb --- /dev/null +++ b/shared/extra-utils/server/debug.ts @@ -0,0 +1,19 @@ +import { makeGetRequest } from '../requests/requests' +import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' + +function getDebug (url: string, token: string) { + const path = '/api/v1/server/debug' + + return makeGetRequest({ + url, + path, + token, + statusCodeExpected: HttpStatusCode.OK_200 + }) +} + +// --------------------------------------------------------------------------- + +export { + getDebug +} diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index a53749589..97971f960 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts @@ -1,7 +1,7 @@ import * as request from 'supertest' import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' -import { makeGetRequest } from '../../../shared/extra-utils' -import { Job, JobState, JobType } from '../../models' +import { getDebug, makeGetRequest } from '../../../shared/extra-utils' +import { Job, JobState, JobType, ServerDebug } from '../../models' import { wait } from '../miscs/miscs' import { ServerInfo } from './servers' @@ -53,7 +53,10 @@ function getJobsListPaginationAndSort (options: { } async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { - const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) : 2000 + const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT + ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) + : 500 + let servers: ServerInfo[] if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ] @@ -75,16 +78,26 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { start: 0, count: 10, sort: '-createdAt' - }) - .then(res => res.body.data) + }).then(res => res.body.data) .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) .then(jobs => { if (jobs.length !== 0) { pendingRequests = true } }) + tasks.push(p) } + + const p = getDebug(server.url, server.accessToken) + .then(res => res.body) + .then((obj: ServerDebug) => { + if (obj.activityPubMessagesWaiting !== 0) { + pendingRequests = true + } + }) + + tasks.push(p) } return tasks diff --git a/shared/models/server/index.ts b/shared/models/server/index.ts index f253a3a79..b5163954a 100644 --- a/shared/models/server/index.ts +++ b/shared/models/server/index.ts @@ -7,5 +7,6 @@ export * from './emailer.model' export * from './job.model' export * from './log-level.type' export * from './server-config.model' +export * from './server-debug.model' export * from './server-error-code.enum' export * from './server-stats.model' diff --git a/shared/models/server/server-debug.model.ts b/shared/models/server/server-debug.model.ts new file mode 100644 index 000000000..4b731bb90 --- /dev/null +++ b/shared/models/server/server-debug.model.ts @@ -0,0 +1,4 @@ +export interface ServerDebug { + ip: string + activityPubMessagesWaiting: number +} -- 2.41.0