From 1061c73fde3005100ead8764eacb444f240440d6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Dec 2019 14:49:59 +0100 Subject: [PATCH] Add ability to filter per job type --- .../src/app/+admin/system/jobs/job.service.ts | 7 ++- .../+admin/system/jobs/jobs.component.html | 20 ++++++-- .../+admin/system/jobs/jobs.component.scss | 18 ++++++- .../app/+admin/system/jobs/jobs.component.ts | 39 +++++++++++---- client/src/types/job-type-client.type.ts | 3 ++ server/controllers/api/jobs.ts | 11 +++- server/helpers/custom-validators/jobs.ts | 10 +++- server/lib/job-queue/job-queue.ts | 26 ++++++++-- server/middlewares/validators/jobs.ts | 12 +++-- server/tests/api/check-params/jobs.ts | 12 +++++ server/tests/api/server/handle-down.ts | 9 +++- server/tests/api/server/jobs.ts | 50 ++++++++++++++----- server/tests/real-world/real-world.ts | 9 +++- shared/extra-utils/server/jobs.ts | 46 ++++++++++++----- shared/models/server/job.model.ts | 6 +-- 15 files changed, 220 insertions(+), 58 deletions(-) create mode 100644 client/src/types/job-type-client.type.ts diff --git a/client/src/app/+admin/system/jobs/job.service.ts b/client/src/app/+admin/system/jobs/job.service.ts index 1daae8f03..120144dff 100644 --- a/client/src/app/+admin/system/jobs/job.service.ts +++ b/client/src/app/+admin/system/jobs/job.service.ts @@ -3,11 +3,12 @@ import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { SortMeta } from 'primeng/api' import { Observable } from 'rxjs' -import { ResultList } from '../../../../../../shared' +import { JobType, ResultList } from '../../../../../../shared' import { JobState } from '../../../../../../shared/models' import { Job } from '../../../../../../shared/models/server/job.model' import { environment } from '../../../../environments/environment' import { RestExtractor, RestPagination, RestService } from '../../../shared' +import { JobTypeClient } from '../../../../types/job-type-client.type' @Injectable() export class JobService { @@ -19,10 +20,12 @@ export class JobService { private restExtractor: RestExtractor ) {} - getJobs (state: JobState, pagination: RestPagination, sort: SortMeta): Observable> { + getJobs (state: JobState, jobType: JobTypeClient, pagination: RestPagination, sort: SortMeta): Observable> { let params = new HttpParams() params = this.restService.addRestGetParams(params, pagination, sort) + if (jobType !== 'all') params = params.append('jobType', jobType) + return this.authHttp.get>(JobService.BASE_JOB_URL + '/' + state, { params }) .pipe( map(res => { diff --git a/client/src/app/+admin/system/jobs/jobs.component.html b/client/src/app/+admin/system/jobs/jobs.component.html index 7ed1888e2..cd26257dd 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.html +++ b/client/src/app/+admin/system/jobs/jobs.component.html @@ -1,10 +1,22 @@
Jobs list
-
- +
+ +
+ +
+
+ +
+ +
+ +
diff --git a/client/src/app/+admin/system/jobs/jobs.component.scss b/client/src/app/+admin/system/jobs/jobs.component.scss index ab05f1982..ccc0b35ca 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.scss +++ b/client/src/app/+admin/system/jobs/jobs.component.scss @@ -1,8 +1,22 @@ @import '_variables'; @import '_mixins'; -.peertube-select-container { - @include peertube-select-container(auto); +.admin-sub-header { + align-items: flex-end; + + .select-filter-block { + &:not(:last-child) { + margin-right: 10px; + } + + label { + margin-bottom: 2px; + } + + .peertube-select-container { + @include peertube-select-container(auto); + } + } } pre { diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index b24353ca6..95ee17023 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts @@ -2,11 +2,12 @@ import { Component, OnInit } from '@angular/core' import { peertubeLocalStorage } from '@app/shared/misc/peertube-local-storage' import { Notifier } from '@app/core' import { SortMeta } from 'primeng/api' -import { Job } from '../../../../../../shared/index' +import { Job, JobType } from '../../../../../../shared/index' import { JobState } from '../../../../../../shared/models' import { RestPagination, RestTable } from '../../../shared' import { JobService } from './job.service' import { I18n } from '@ngx-translate/i18n-polyfill' +import { JobTypeClient } from '../../../../types/job-type-client.type' @Component({ selector: 'my-jobs', @@ -15,9 +16,26 @@ import { I18n } from '@ngx-translate/i18n-polyfill' }) export class JobsComponent extends RestTable implements OnInit { private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state' + private static JOB_STATE_LOCAL_STORAGE_TYPE = 'jobs-list-type' jobState: JobState = 'waiting' jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] + + jobType: JobTypeClient = 'all' + jobTypes: JobTypeClient[] = [ + 'all', + 'activitypub-follow', + 'activitypub-http-broadcast', + 'activitypub-http-fetcher', + 'activitypub-http-unicast', + 'email', + 'video-transcoding', + 'video-file-import', + 'video-import', + 'videos-views', + 'activitypub-refresher' + ] + jobs: Job[] = [] totalRecords: number rowsPerPage = 10 @@ -33,20 +51,20 @@ export class JobsComponent extends RestTable implements OnInit { } ngOnInit () { - this.loadJobState() + this.loadJobStateAndType() this.initialize() } - onJobStateChanged () { + onJobStateOrTypeChanged () { this.pagination.start = 0 this.loadData() - this.saveJobState() + this.saveJobStateAndType() } protected loadData () { this.jobsService - .getJobs(this.jobState, this.pagination, this.sort) + .getJobs(this.jobState, this.jobType, this.pagination, this.sort) .subscribe( resultList => { this.jobs = resultList.data @@ -57,13 +75,16 @@ export class JobsComponent extends RestTable implements OnInit { ) } - private loadJobState () { - const result = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE) + private loadJobStateAndType () { + const state = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE) + if (state) this.jobState = state as JobState - if (result) this.jobState = result as JobState + const type = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE) + if (type) this.jobType = type as JobType } - private saveJobState () { + private saveJobStateAndType () { peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState) + peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE, this.jobType) } } diff --git a/client/src/types/job-type-client.type.ts b/client/src/types/job-type-client.type.ts new file mode 100644 index 000000000..7d51f1db2 --- /dev/null +++ b/client/src/types/job-type-client.type.ts @@ -0,0 +1,3 @@ +import { JobType } from '@shared/models' + +export type JobTypeClient = 'all' | JobType diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts index 1fa662349..05320311e 100644 --- a/server/controllers/api/jobs.ts +++ b/server/controllers/api/jobs.ts @@ -24,7 +24,7 @@ jobsRouter.get('/:state', jobsSortValidator, setDefaultSort, setDefaultPagination, - asyncMiddleware(listJobsValidator), + listJobsValidator, asyncMiddleware(listJobs) ) @@ -39,8 +39,15 @@ export { async function listJobs (req: express.Request, res: express.Response) { const state = req.params.state as JobState const asc = req.query.sort === 'createdAt' + const jobType = req.query.jobType - const jobs = await JobQueue.Instance.listForApi(state, req.query.start, req.query.count, asc) + const jobs = await JobQueue.Instance.listForApi({ + state, + start: req.query.start, + count: req.query.count, + asc, + jobType + }) const total = await JobQueue.Instance.count(state) const result: ResultList = { diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts index 1cc6e6912..dd33e85a3 100644 --- a/server/helpers/custom-validators/jobs.ts +++ b/server/helpers/custom-validators/jobs.ts @@ -1,14 +1,20 @@ import { JobState } from '../../../shared/models' import { exists } from './misc' +import { jobTypes } from '@server/lib/job-queue/job-queue' const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] function isValidJobState (value: JobState) { - return exists(value) && jobStates.indexOf(value) !== -1 + return exists(value) && jobStates.includes(value) +} + +function isValidJobType (value: any) { + return exists(value) && jobTypes.includes(value) } // --------------------------------------------------------------------------- export { - isValidJobState + isValidJobState, + isValidJobType } diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 3c810da98..ec601e9ea 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -121,11 +121,20 @@ class JobQueue { return queue.add(obj.payload, jobArgs) } - async listForApi (state: JobState, start: number, count: number, asc?: boolean): Promise { + async listForApi (options: { + state: JobState, + start: number, + count: number, + asc?: boolean, + jobType: JobType + }): Promise { + const { state, start, count, asc, jobType } = options let results: Bull.Job[] = [] + const filteredJobTypes = this.filterJobTypes(jobType) + // TODO: optimize - for (const jobType of jobTypes) { + for (const jobType of filteredJobTypes) { const queue = this.queues[ jobType ] if (queue === undefined) { logger.error('Unknown queue %s to list jobs.', jobType) @@ -149,10 +158,12 @@ class JobQueue { return results.slice(start, start + count) } - async count (state: JobState): Promise { + async count (state: JobState, jobType?: JobType): Promise { let total = 0 - for (const type of jobTypes) { + const filteredJobTypes = this.filterJobTypes(jobType) + + for (const type of filteredJobTypes) { const queue = this.queues[ type ] if (queue === undefined) { logger.error('Unknown queue %s to count jobs.', type) @@ -180,6 +191,12 @@ class JobQueue { }) } + private filterJobTypes (jobType?: JobType) { + if (!jobType) return jobTypes + + return jobTypes.filter(t => t === jobType) + } + static get Instance () { return this.instance || (this.instance = new this()) } @@ -188,5 +205,6 @@ class JobQueue { // --------------------------------------------------------------------------- export { + jobTypes, JobQueue } diff --git a/server/middlewares/validators/jobs.ts b/server/middlewares/validators/jobs.ts index 41a8d6899..b57615dbc 100644 --- a/server/middlewares/validators/jobs.ts +++ b/server/middlewares/validators/jobs.ts @@ -1,13 +1,17 @@ import * as express from 'express' -import { param } from 'express-validator' -import { isValidJobState } from '../../helpers/custom-validators/jobs' +import { param, query } from 'express-validator' +import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs' import { logger } from '../../helpers/logger' import { areValidationErrors } from './utils' const listJobsValidator = [ - param('state').custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'), + param('state') + .custom(isValidJobState).not().isEmpty().withMessage('Should have a valid job state'), + query('jobType') + .optional() + .custom(isValidJobType).withMessage('Should have a valid job state'), - async (req: express.Request, res: express.Response, next: express.NextFunction) => { + (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking listJobsValidator parameters.', { parameters: req.params }) if (areValidationErrors(req, res)) return diff --git a/server/tests/api/check-params/jobs.ts b/server/tests/api/check-params/jobs.ts index c70139514..22e237964 100644 --- a/server/tests/api/check-params/jobs.ts +++ b/server/tests/api/check-params/jobs.ts @@ -51,6 +51,17 @@ describe('Test jobs API validators', function () { }) }) + it('Should fail with an incorrect job type', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { + jobType: 'toto' + } + }) + }) + it('Should fail with a bad start pagination', async function () { await checkBadStartPagination(server.url, path, server.accessToken) }) @@ -79,6 +90,7 @@ describe('Test jobs API validators', function () { statusCodeExpected: 403 }) }) + }) after(async function () { diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index a0f505474..7e36067f1 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -184,7 +184,14 @@ describe('Test handle downs', function () { const states: JobState[] = [ 'waiting', 'active' ] for (const state of states) { - const res = await getJobsListPaginationAndSort(servers[ 0 ].url, servers[ 0 ].accessToken, state,0, 50, '-createdAt') + const res = await getJobsListPaginationAndSort({ + url: servers[ 0 ].url, + accessToken: servers[ 0 ].accessToken, + state: state, + start: 0, + count: 50, + sort: '-createdAt' + }) expect(res.body.data).to.have.length(0) } }) diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index ceea47a85..58d8c8c10 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts @@ -41,20 +41,46 @@ describe('Test jobs', function () { expect(res.body.data).to.have.length.above(2) }) - it('Should list jobs with sort and pagination', async function () { - const res = await getJobsListPaginationAndSort(servers[1].url, servers[1].accessToken, 'completed', 1, 2, 'createdAt') - expect(res.body.total).to.be.above(2) - expect(res.body.data).to.have.lengthOf(2) + it('Should list jobs with sort, pagination and job type', async function () { + { + const res = await getJobsListPaginationAndSort({ + url: servers[ 1 ].url, + accessToken: servers[ 1 ].accessToken, + state: 'completed', + start: 1, + count: 2, + sort: 'createdAt' + }) + expect(res.body.total).to.be.above(2) + expect(res.body.data).to.have.lengthOf(2) + + let job: Job = res.body.data[ 0 ] + // Skip repeat jobs + if (job.type === 'videos-views') job = res.body.data[ 1 ] + + expect(job.state).to.equal('completed') + expect(job.type.startsWith('activitypub-')).to.be.true + expect(dateIsValid(job.createdAt as string)).to.be.true + expect(dateIsValid(job.processedOn as string)).to.be.true + expect(dateIsValid(job.finishedOn as string)).to.be.true + } - let job = res.body.data[0] - // Skip repeat jobs - if (job.type === 'videos-views') job = res.body.data[1] + { + const res = await getJobsListPaginationAndSort({ + url: servers[ 1 ].url, + accessToken: servers[ 1 ].accessToken, + state: 'completed', + start: 0, + count: 100, + sort: 'createdAt', + jobType: 'activitypub-http-broadcast' + }) + expect(res.body.total).to.be.above(2) - expect(job.state).to.equal('completed') - expect(job.type.startsWith('activitypub-')).to.be.true - expect(dateIsValid(job.createdAt)).to.be.true - expect(dateIsValid(job.processedOn)).to.be.true - expect(dateIsValid(job.finishedOn)).to.be.true + for (const j of res.body.data as Job[]) { + expect(j.type).to.equal('activitypub-http-broadcast') + } + } }) after(async function () { diff --git a/server/tests/real-world/real-world.ts b/server/tests/real-world/real-world.ts index 8b070004d..cba5ac311 100644 --- a/server/tests/real-world/real-world.ts +++ b/server/tests/real-world/real-world.ts @@ -354,7 +354,14 @@ async function isTherePendingRequests (servers: ServerInfo[]) { // Check if each server has pending request for (const server of servers) { for (const state of states) { - const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') + const p = getJobsListPaginationAndSort({ + url: server.url, + accessToken: server.accessToken, + state: state, + start: 0, + count: 10, + sort: '-createdAt' + }) .then(res => { if (res.body.total > 0) pendingRequests = true }) diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index b3db885e8..cc1352e14 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts @@ -1,7 +1,8 @@ import * as request from 'supertest' -import { Job, JobState } from '../../models' +import { Job, JobState, JobType } from '../../models' import { wait } from '../miscs/miscs' import { ServerInfo } from './servers' +import { makeGetRequest } from '@shared/extra-utils' function getJobsList (url: string, accessToken: string, state: JobState) { const path = '/api/v1/jobs/' + state @@ -14,18 +15,32 @@ function getJobsList (url: string, accessToken: string, state: JobState) { .expect('Content-Type', /json/) } -function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) { +function getJobsListPaginationAndSort (options: { + url: string, + accessToken: string, + state: JobState, + start: number, + count: number, + sort: string, + jobType?: JobType +}) { + const { url, accessToken, state, start, count, sort, jobType } = options const path = '/api/v1/jobs/' + state - return request(url) - .get(path) - .query({ start }) - .query({ count }) - .query({ sort }) - .set('Accept', 'application/json') - .set('Authorization', 'Bearer ' + accessToken) - .expect(200) - .expect('Content-Type', /json/) + const query = { + start, + count, + sort, + jobType + } + + return makeGetRequest({ + url, + path, + token: accessToken, + statusCodeExpected: 200, + query + }) } async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { @@ -44,7 +59,14 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { // Check if each server has pending request for (const server of servers) { for (const state of states) { - const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') + const p = getJobsListPaginationAndSort({ + url: server.url, + accessToken: server.accessToken, + state: state, + start: 0, + count: 10, + sort: '-createdAt' + }) .then(res => res.body.data) .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) .then(jobs => { diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 1b9aa8a07..b82a633b2 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -17,7 +17,7 @@ export interface Job { type: JobType data: any, error: any, - createdAt: Date - finishedOn: Date - processedOn: Date + createdAt: Date | string + finishedOn: Date | string + processedOn: Date | string } -- 2.41.0