diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/jobs.ts | 11 | ||||
-rw-r--r-- | server/helpers/custom-validators/jobs.ts | 10 | ||||
-rw-r--r-- | server/lib/job-queue/job-queue.ts | 26 | ||||
-rw-r--r-- | server/middlewares/validators/jobs.ts | 12 | ||||
-rw-r--r-- | server/tests/api/check-params/jobs.ts | 12 | ||||
-rw-r--r-- | server/tests/api/server/handle-down.ts | 9 | ||||
-rw-r--r-- | server/tests/api/server/jobs.ts | 50 | ||||
-rw-r--r-- | server/tests/real-world/real-world.ts | 9 |
8 files changed, 113 insertions, 26 deletions
diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts index 1fa662349..05320311e 100644 --- a/server/controllers/api/jobs.ts +++ b/server/controllers/api/jobs.ts | |||
@@ -24,7 +24,7 @@ jobsRouter.get('/:state', | |||
24 | jobsSortValidator, | 24 | jobsSortValidator, |
25 | setDefaultSort, | 25 | setDefaultSort, |
26 | setDefaultPagination, | 26 | setDefaultPagination, |
27 | asyncMiddleware(listJobsValidator), | 27 | listJobsValidator, |
28 | asyncMiddleware(listJobs) | 28 | asyncMiddleware(listJobs) |
29 | ) | 29 | ) |
30 | 30 | ||
@@ -39,8 +39,15 @@ export { | |||
39 | async function listJobs (req: express.Request, res: express.Response) { | 39 | async function listJobs (req: express.Request, res: express.Response) { |
40 | const state = req.params.state as JobState | 40 | const state = req.params.state as JobState |
41 | const asc = req.query.sort === 'createdAt' | 41 | const asc = req.query.sort === 'createdAt' |
42 | const jobType = req.query.jobType | ||
42 | 43 | ||
43 | const jobs = await JobQueue.Instance.listForApi(state, req.query.start, req.query.count, asc) | 44 | const jobs = await JobQueue.Instance.listForApi({ |
45 | state, | ||
46 | start: req.query.start, | ||
47 | count: req.query.count, | ||
48 | asc, | ||
49 | jobType | ||
50 | }) | ||
44 | const total = await JobQueue.Instance.count(state) | 51 | const total = await JobQueue.Instance.count(state) |
45 | 52 | ||
46 | const result: ResultList<any> = { | 53 | const result: ResultList<any> = { |
diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts index 1cc6e6912..dd33e85a3 100644 --- a/server/helpers/custom-validators/jobs.ts +++ b/server/helpers/custom-validators/jobs.ts | |||
@@ -1,14 +1,20 @@ | |||
1 | import { JobState } from '../../../shared/models' | 1 | import { JobState } from '../../../shared/models' |
2 | import { exists } from './misc' | 2 | import { exists } from './misc' |
3 | import { jobTypes } from '@server/lib/job-queue/job-queue' | ||
3 | 4 | ||
4 | const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] | 5 | const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] |
5 | 6 | ||
6 | function isValidJobState (value: JobState) { | 7 | function isValidJobState (value: JobState) { |
7 | return exists(value) && jobStates.indexOf(value) !== -1 | 8 | return exists(value) && jobStates.includes(value) |
9 | } | ||
10 | |||
11 | function isValidJobType (value: any) { | ||
12 | return exists(value) && jobTypes.includes(value) | ||
8 | } | 13 | } |
9 | 14 | ||
10 | // --------------------------------------------------------------------------- | 15 | // --------------------------------------------------------------------------- |
11 | 16 | ||
12 | export { | 17 | export { |
13 | isValidJobState | 18 | isValidJobState, |
19 | isValidJobType | ||
14 | } | 20 | } |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 3c810da98..ec601e9ea 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -121,11 +121,20 @@ class JobQueue { | |||
121 | return queue.add(obj.payload, jobArgs) | 121 | return queue.add(obj.payload, jobArgs) |
122 | } | 122 | } |
123 | 123 | ||
124 | async listForApi (state: JobState, start: number, count: number, asc?: boolean): Promise<Bull.Job[]> { | 124 | async listForApi (options: { |
125 | state: JobState, | ||
126 | start: number, | ||
127 | count: number, | ||
128 | asc?: boolean, | ||
129 | jobType: JobType | ||
130 | }): Promise<Bull.Job[]> { | ||
131 | const { state, start, count, asc, jobType } = options | ||
125 | let results: Bull.Job[] = [] | 132 | let results: Bull.Job[] = [] |
126 | 133 | ||
134 | const filteredJobTypes = this.filterJobTypes(jobType) | ||
135 | |||
127 | // TODO: optimize | 136 | // TODO: optimize |
128 | for (const jobType of jobTypes) { | 137 | for (const jobType of filteredJobTypes) { |
129 | const queue = this.queues[ jobType ] | 138 | const queue = this.queues[ jobType ] |
130 | if (queue === undefined) { | 139 | if (queue === undefined) { |
131 | logger.error('Unknown queue %s to list jobs.', jobType) | 140 | logger.error('Unknown queue %s to list jobs.', jobType) |
@@ -149,10 +158,12 @@ class JobQueue { | |||
149 | return results.slice(start, start + count) | 158 | return results.slice(start, start + count) |
150 | } | 159 | } |
151 | 160 | ||
152 | async count (state: JobState): Promise<number> { | 161 | async count (state: JobState, jobType?: JobType): Promise<number> { |
153 | let total = 0 | 162 | let total = 0 |
154 | 163 | ||
155 | for (const type of jobTypes) { | 164 | const filteredJobTypes = this.filterJobTypes(jobType) |
165 | |||
166 | for (const type of filteredJobTypes) { | ||
156 | const queue = this.queues[ type ] | 167 | const queue = this.queues[ type ] |
157 | if (queue === undefined) { | 168 | if (queue === undefined) { |
158 | logger.error('Unknown queue %s to count jobs.', type) | 169 | logger.error('Unknown queue %s to count jobs.', type) |
@@ -180,6 +191,12 @@ class JobQueue { | |||
180 | }) | 191 | }) |
181 | } | 192 | } |
182 | 193 | ||
194 | private filterJobTypes (jobType?: JobType) { | ||
195 | if (!jobType) return jobTypes | ||
196 | |||
197 | return jobTypes.filter(t => t === jobType) | ||
198 | } | ||
199 | |||
183 | static get Instance () { | 200 | static get Instance () { |
184 | return this.instance || (this.instance = new this()) | 201 | return this.instance || (this.instance = new this()) |
185 | } | 202 | } |
@@ -188,5 +205,6 @@ class JobQueue { | |||
188 | // --------------------------------------------------------------------------- | 205 | // --------------------------------------------------------------------------- |
189 | 206 | ||
190 | export { | 207 | export { |
208 | jobTypes, | ||
191 | JobQueue | 209 | JobQueue |
192 | } | 210 | } |
diff --git a/server/middlewares/validators/jobs.ts b/server/middlewares/validators/jobs.ts index 41a8d6899..b57615dbc 100644 --- a/server/middlewares/validators/jobs.ts +++ b/server/middlewares/validators/jobs.ts | |||
@@ -1,13 +1,17 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { param } from 'express-validator' | 2 | import { param, query } from 'express-validator' |
3 | import { isValidJobState } from '../../helpers/custom-validators/jobs' | 3 | import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs' |
4 | import { logger } from '../../helpers/logger' | 4 | import { logger } from '../../helpers/logger' |
5 | import { areValidationErrors } from './utils' | 5 | import { areValidationErrors } from './utils' |
6 | 6 | ||
7 | const listJobsValidator = [ | 7 | const listJobsValidator = [ |
8 | param('state').custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'), | 8 | param('state') |
9 | .custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'), | ||
10 | query('jobType') | ||
11 | .optional() | ||
12 | .custom(isValidJobType).withMessage('Should have a valid job state'), | ||
9 | 13 | ||
10 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 14 | (req: express.Request, res: express.Response, next: express.NextFunction) => { |
11 | logger.debug('Checking listJobsValidator parameters.', { parameters: req.params }) | 15 | logger.debug('Checking listJobsValidator parameters.', { parameters: req.params }) |
12 | 16 | ||
13 | if (areValidationErrors(req, res)) return | 17 | if (areValidationErrors(req, res)) return |
diff --git a/server/tests/api/check-params/jobs.ts b/server/tests/api/check-params/jobs.ts index c70139514..22e237964 100644 --- a/server/tests/api/check-params/jobs.ts +++ b/server/tests/api/check-params/jobs.ts | |||
@@ -51,6 +51,17 @@ describe('Test jobs API validators', function () { | |||
51 | }) | 51 | }) |
52 | }) | 52 | }) |
53 | 53 | ||
54 | it('Should fail with an incorrect job type', async function () { | ||
55 | await makeGetRequest({ | ||
56 | url: server.url, | ||
57 | token: server.accessToken, | ||
58 | path, | ||
59 | query: { | ||
60 | jobType: 'toto' | ||
61 | } | ||
62 | }) | ||
63 | }) | ||
64 | |||
54 | it('Should fail with a bad start pagination', async function () { | 65 | it('Should fail with a bad start pagination', async function () { |
55 | await checkBadStartPagination(server.url, path, server.accessToken) | 66 | await checkBadStartPagination(server.url, path, server.accessToken) |
56 | }) | 67 | }) |
@@ -79,6 +90,7 @@ describe('Test jobs API validators', function () { | |||
79 | statusCodeExpected: 403 | 90 | statusCodeExpected: 403 |
80 | }) | 91 | }) |
81 | }) | 92 | }) |
93 | |||
82 | }) | 94 | }) |
83 | 95 | ||
84 | after(async function () { | 96 | after(async function () { |
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index a0f505474..7e36067f1 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts | |||
@@ -184,7 +184,14 @@ describe('Test handle downs', function () { | |||
184 | const states: JobState[] = [ 'waiting', 'active' ] | 184 | const states: JobState[] = [ 'waiting', 'active' ] |
185 | 185 | ||
186 | for (const state of states) { | 186 | for (const state of states) { |
187 | const res = await getJobsListPaginationAndSort(servers[ 0 ].url, servers[ 0 ].accessToken, state,0, 50, '-createdAt') | 187 | const res = await getJobsListPaginationAndSort({ |
188 | url: servers[ 0 ].url, | ||
189 | accessToken: servers[ 0 ].accessToken, | ||
190 | state: state, | ||
191 | start: 0, | ||
192 | count: 50, | ||
193 | sort: '-createdAt' | ||
194 | }) | ||
188 | expect(res.body.data).to.have.length(0) | 195 | expect(res.body.data).to.have.length(0) |
189 | } | 196 | } |
190 | }) | 197 | }) |
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index ceea47a85..58d8c8c10 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts | |||
@@ -41,20 +41,46 @@ describe('Test jobs', function () { | |||
41 | expect(res.body.data).to.have.length.above(2) | 41 | expect(res.body.data).to.have.length.above(2) |
42 | }) | 42 | }) |
43 | 43 | ||
44 | it('Should list jobs with sort and pagination', async function () { | 44 | it('Should list jobs with sort, pagination and job type', async function () { |
45 | const res = await getJobsListPaginationAndSort(servers[1].url, servers[1].accessToken, 'completed', 1, 2, 'createdAt') | 45 | { |
46 | expect(res.body.total).to.be.above(2) | 46 | const res = await getJobsListPaginationAndSort({ |
47 | expect(res.body.data).to.have.lengthOf(2) | 47 | url: servers[ 1 ].url, |
48 | accessToken: servers[ 1 ].accessToken, | ||
49 | state: 'completed', | ||
50 | start: 1, | ||
51 | count: 2, | ||
52 | sort: 'createdAt' | ||
53 | }) | ||
54 | expect(res.body.total).to.be.above(2) | ||
55 | expect(res.body.data).to.have.lengthOf(2) | ||
56 | |||
57 | let job: Job = res.body.data[ 0 ] | ||
58 | // Skip repeat jobs | ||
59 | if (job.type === 'videos-views') job = res.body.data[ 1 ] | ||
60 | |||
61 | expect(job.state).to.equal('completed') | ||
62 | expect(job.type.startsWith('activitypub-')).to.be.true | ||
63 | expect(dateIsValid(job.createdAt as string)).to.be.true | ||
64 | expect(dateIsValid(job.processedOn as string)).to.be.true | ||
65 | expect(dateIsValid(job.finishedOn as string)).to.be.true | ||
66 | } | ||
48 | 67 | ||
49 | let job = res.body.data[0] | 68 | { |
50 | // Skip repeat jobs | 69 | const res = await getJobsListPaginationAndSort({ |
51 | if (job.type === 'videos-views') job = res.body.data[1] | 70 | url: servers[ 1 ].url, |
71 | accessToken: servers[ 1 ].accessToken, | ||
72 | state: 'completed', | ||
73 | start: 0, | ||
74 | count: 100, | ||
75 | sort: 'createdAt', | ||
76 | jobType: 'activitypub-http-broadcast' | ||
77 | }) | ||
78 | expect(res.body.total).to.be.above(2) | ||
52 | 79 | ||
53 | expect(job.state).to.equal('completed') | 80 | for (const j of res.body.data as Job[]) { |
54 | expect(job.type.startsWith('activitypub-')).to.be.true | 81 | expect(j.type).to.equal('activitypub-http-broadcast') |
55 | expect(dateIsValid(job.createdAt)).to.be.true | 82 | } |
56 | expect(dateIsValid(job.processedOn)).to.be.true | 83 | } |
57 | expect(dateIsValid(job.finishedOn)).to.be.true | ||
58 | }) | 84 | }) |
59 | 85 | ||
60 | after(async function () { | 86 | after(async function () { |
diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts index 8b070004d..cba5ac311 100644 --- a/server/tests/real-world/real-world.ts +++ b/server/tests/real-world/real-world.ts | |||
@@ -354,7 +354,14 @@ async function isTherePendingRequests (servers: ServerInfo[]) { | |||
354 | // Check if each server has pending request | 354 | // Check if each server has pending request |
355 | for (const server of servers) { | 355 | for (const server of servers) { |
356 | for (const state of states) { | 356 | for (const state of states) { |
357 | const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') | 357 | const p = getJobsListPaginationAndSort({ |
358 | url: server.url, | ||
359 | accessToken: server.accessToken, | ||
360 | state: state, | ||
361 | start: 0, | ||
362 | count: 10, | ||
363 | sort: '-createdAt' | ||
364 | }) | ||
358 | .then(res => { | 365 | .then(res => { |
359 | if (res.body.total > 0) pendingRequests = true | 366 | if (res.body.total > 0) pendingRequests = true |
360 | }) | 367 | }) |