aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/jobs.ts11
-rw-r--r--server/helpers/custom-validators/jobs.ts10
-rw-r--r--server/lib/job-queue/job-queue.ts26
-rw-r--r--server/middlewares/validators/jobs.ts12
-rw-r--r--server/tests/api/check-params/jobs.ts12
-rw-r--r--server/tests/api/server/handle-down.ts9
-rw-r--r--server/tests/api/server/jobs.ts50
-rw-r--r--server/tests/real-world/real-world.ts9
8 files changed, 113 insertions, 26 deletions
diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts
index 1fa662349..05320311e 100644
--- a/server/controllers/api/jobs.ts
+++ b/server/controllers/api/jobs.ts
@@ -24,7 +24,7 @@ jobsRouter.get('/:state',
24 jobsSortValidator, 24 jobsSortValidator,
25 setDefaultSort, 25 setDefaultSort,
26 setDefaultPagination, 26 setDefaultPagination,
27 asyncMiddleware(listJobsValidator), 27 listJobsValidator,
28 asyncMiddleware(listJobs) 28 asyncMiddleware(listJobs)
29) 29)
30 30
@@ -39,8 +39,15 @@ export {
39async function listJobs (req: express.Request, res: express.Response) { 39async function listJobs (req: express.Request, res: express.Response) {
40 const state = req.params.state as JobState 40 const state = req.params.state as JobState
41 const asc = req.query.sort === 'createdAt' 41 const asc = req.query.sort === 'createdAt'
42 const jobType = req.query.jobType
42 43
43 const jobs = await JobQueue.Instance.listForApi(state, req.query.start, req.query.count, asc) 44 const jobs = await JobQueue.Instance.listForApi({
45 state,
46 start: req.query.start,
47 count: req.query.count,
48 asc,
49 jobType
50 })
44 const total = await JobQueue.Instance.count(state) 51 const total = await JobQueue.Instance.count(state)
45 52
46 const result: ResultList<any> = { 53 const result: ResultList<any> = {
diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts
index 1cc6e6912..dd33e85a3 100644
--- a/server/helpers/custom-validators/jobs.ts
+++ b/server/helpers/custom-validators/jobs.ts
@@ -1,14 +1,20 @@
1import { JobState } from '../../../shared/models' 1import { JobState } from '../../../shared/models'
2import { exists } from './misc' 2import { exists } from './misc'
3import { jobTypes } from '@server/lib/job-queue/job-queue'
3 4
4const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] 5const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ]
5 6
6function isValidJobState (value: JobState) { 7function isValidJobState (value: JobState) {
7 return exists(value) && jobStates.indexOf(value) !== -1 8 return exists(value) && jobStates.includes(value)
9}
10
11function isValidJobType (value: any) {
12 return exists(value) && jobTypes.includes(value)
8} 13}
9 14
10// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
11 16
12export { 17export {
13 isValidJobState 18 isValidJobState,
19 isValidJobType
14} 20}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 3c810da98..ec601e9ea 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -121,11 +121,20 @@ class JobQueue {
121 return queue.add(obj.payload, jobArgs) 121 return queue.add(obj.payload, jobArgs)
122 } 122 }
123 123
124 async listForApi (state: JobState, start: number, count: number, asc?: boolean): Promise<Bull.Job[]> { 124 async listForApi (options: {
125 state: JobState,
126 start: number,
127 count: number,
128 asc?: boolean,
129 jobType: JobType
130 }): Promise<Bull.Job[]> {
131 const { state, start, count, asc, jobType } = options
125 let results: Bull.Job[] = [] 132 let results: Bull.Job[] = []
126 133
134 const filteredJobTypes = this.filterJobTypes(jobType)
135
127 // TODO: optimize 136 // TODO: optimize
128 for (const jobType of jobTypes) { 137 for (const jobType of filteredJobTypes) {
129 const queue = this.queues[ jobType ] 138 const queue = this.queues[ jobType ]
130 if (queue === undefined) { 139 if (queue === undefined) {
131 logger.error('Unknown queue %s to list jobs.', jobType) 140 logger.error('Unknown queue %s to list jobs.', jobType)
@@ -149,10 +158,12 @@ class JobQueue {
149 return results.slice(start, start + count) 158 return results.slice(start, start + count)
150 } 159 }
151 160
152 async count (state: JobState): Promise<number> { 161 async count (state: JobState, jobType?: JobType): Promise<number> {
153 let total = 0 162 let total = 0
154 163
155 for (const type of jobTypes) { 164 const filteredJobTypes = this.filterJobTypes(jobType)
165
166 for (const type of filteredJobTypes) {
156 const queue = this.queues[ type ] 167 const queue = this.queues[ type ]
157 if (queue === undefined) { 168 if (queue === undefined) {
158 logger.error('Unknown queue %s to count jobs.', type) 169 logger.error('Unknown queue %s to count jobs.', type)
@@ -180,6 +191,12 @@ class JobQueue {
180 }) 191 })
181 } 192 }
182 193
194 private filterJobTypes (jobType?: JobType) {
195 if (!jobType) return jobTypes
196
197 return jobTypes.filter(t => t === jobType)
198 }
199
183 static get Instance () { 200 static get Instance () {
184 return this.instance || (this.instance = new this()) 201 return this.instance || (this.instance = new this())
185 } 202 }
@@ -188,5 +205,6 @@ class JobQueue {
188// --------------------------------------------------------------------------- 205// ---------------------------------------------------------------------------
189 206
190export { 207export {
208 jobTypes,
191 JobQueue 209 JobQueue
192} 210}
diff --git a/server/middlewares/validators/jobs.ts b/server/middlewares/validators/jobs.ts
index 41a8d6899..b57615dbc 100644
--- a/server/middlewares/validators/jobs.ts
+++ b/server/middlewares/validators/jobs.ts
@@ -1,13 +1,17 @@
1import * as express from 'express' 1import * as express from 'express'
2import { param } from 'express-validator' 2import { param, query } from 'express-validator'
3import { isValidJobState } from '../../helpers/custom-validators/jobs' 3import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs'
4import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
5import { areValidationErrors } from './utils' 5import { areValidationErrors } from './utils'
6 6
7const listJobsValidator = [ 7const listJobsValidator = [
8 param('state').custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'), 8 param('state')
9 .custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'),
10 query('jobType')
11 .optional()
12 .custom(isValidJobType).withMessage('Should have a valid job state'),
9 13
10 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 14 (req: express.Request, res: express.Response, next: express.NextFunction) => {
11 logger.debug('Checking listJobsValidator parameters.', { parameters: req.params }) 15 logger.debug('Checking listJobsValidator parameters.', { parameters: req.params })
12 16
13 if (areValidationErrors(req, res)) return 17 if (areValidationErrors(req, res)) return
diff --git a/server/tests/api/check-params/jobs.ts b/server/tests/api/check-params/jobs.ts
index c70139514..22e237964 100644
--- a/server/tests/api/check-params/jobs.ts
+++ b/server/tests/api/check-params/jobs.ts
@@ -51,6 +51,17 @@ describe('Test jobs API validators', function () {
51 }) 51 })
52 }) 52 })
53 53
54 it('Should fail with an incorrect job type', async function () {
55 await makeGetRequest({
56 url: server.url,
57 token: server.accessToken,
58 path,
59 query: {
60 jobType: 'toto'
61 }
62 })
63 })
64
54 it('Should fail with a bad start pagination', async function () { 65 it('Should fail with a bad start pagination', async function () {
55 await checkBadStartPagination(server.url, path, server.accessToken) 66 await checkBadStartPagination(server.url, path, server.accessToken)
56 }) 67 })
@@ -79,6 +90,7 @@ describe('Test jobs API validators', function () {
79 statusCodeExpected: 403 90 statusCodeExpected: 403
80 }) 91 })
81 }) 92 })
93
82 }) 94 })
83 95
84 after(async function () { 96 after(async function () {
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index a0f505474..7e36067f1 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -184,7 +184,14 @@ describe('Test handle downs', function () {
184 const states: JobState[] = [ 'waiting', 'active' ] 184 const states: JobState[] = [ 'waiting', 'active' ]
185 185
186 for (const state of states) { 186 for (const state of states) {
187 const res = await getJobsListPaginationAndSort(servers[ 0 ].url, servers[ 0 ].accessToken, state,0, 50, '-createdAt') 187 const res = await getJobsListPaginationAndSort({
188 url: servers[ 0 ].url,
189 accessToken: servers[ 0 ].accessToken,
190 state: state,
191 start: 0,
192 count: 50,
193 sort: '-createdAt'
194 })
188 expect(res.body.data).to.have.length(0) 195 expect(res.body.data).to.have.length(0)
189 } 196 }
190 }) 197 })
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts
index ceea47a85..58d8c8c10 100644
--- a/server/tests/api/server/jobs.ts
+++ b/server/tests/api/server/jobs.ts
@@ -41,20 +41,46 @@ describe('Test jobs', function () {
41 expect(res.body.data).to.have.length.above(2) 41 expect(res.body.data).to.have.length.above(2)
42 }) 42 })
43 43
44 it('Should list jobs with sort and pagination', async function () { 44 it('Should list jobs with sort, pagination and job type', async function () {
45 const res = await getJobsListPaginationAndSort(servers[1].url, servers[1].accessToken, 'completed', 1, 2, 'createdAt') 45 {
46 expect(res.body.total).to.be.above(2) 46 const res = await getJobsListPaginationAndSort({
47 expect(res.body.data).to.have.lengthOf(2) 47 url: servers[ 1 ].url,
48 accessToken: servers[ 1 ].accessToken,
49 state: 'completed',
50 start: 1,
51 count: 2,
52 sort: 'createdAt'
53 })
54 expect(res.body.total).to.be.above(2)
55 expect(res.body.data).to.have.lengthOf(2)
56
57 let job: Job = res.body.data[ 0 ]
58 // Skip repeat jobs
59 if (job.type === 'videos-views') job = res.body.data[ 1 ]
60
61 expect(job.state).to.equal('completed')
62 expect(job.type.startsWith('activitypub-')).to.be.true
63 expect(dateIsValid(job.createdAt as string)).to.be.true
64 expect(dateIsValid(job.processedOn as string)).to.be.true
65 expect(dateIsValid(job.finishedOn as string)).to.be.true
66 }
48 67
49 let job = res.body.data[0] 68 {
50 // Skip repeat jobs 69 const res = await getJobsListPaginationAndSort({
51 if (job.type === 'videos-views') job = res.body.data[1] 70 url: servers[ 1 ].url,
71 accessToken: servers[ 1 ].accessToken,
72 state: 'completed',
73 start: 0,
74 count: 100,
75 sort: 'createdAt',
76 jobType: 'activitypub-http-broadcast'
77 })
78 expect(res.body.total).to.be.above(2)
52 79
53 expect(job.state).to.equal('completed') 80 for (const j of res.body.data as Job[]) {
54 expect(job.type.startsWith('activitypub-')).to.be.true 81 expect(j.type).to.equal('activitypub-http-broadcast')
55 expect(dateIsValid(job.createdAt)).to.be.true 82 }
56 expect(dateIsValid(job.processedOn)).to.be.true 83 }
57 expect(dateIsValid(job.finishedOn)).to.be.true
58 }) 84 })
59 85
60 after(async function () { 86 after(async function () {
diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts
index 8b070004d..cba5ac311 100644
--- a/server/tests/real-world/real-world.ts
+++ b/server/tests/real-world/real-world.ts
@@ -354,7 +354,14 @@ async function isTherePendingRequests (servers: ServerInfo[]) {
354 // Check if each server has pending request 354 // Check if each server has pending request
355 for (const server of servers) { 355 for (const server of servers) {
356 for (const state of states) { 356 for (const state of states) {
357 const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') 357 const p = getJobsListPaginationAndSort({
358 url: server.url,
359 accessToken: server.accessToken,
360 state: state,
361 start: 0,
362 count: 10,
363 sort: '-createdAt'
364 })
358 .then(res => { 365 .then(res => {
359 if (res.body.total > 0) pendingRequests = true 366 if (res.body.total > 0) pendingRequests = true
360 }) 367 })