diff options
author | Chocobozzz <me@florianbigard.com> | 2021-07-07 09:34:56 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2021-07-20 15:27:17 +0200 |
commit | 9c6327f803aaf4200672f1fc40b2f43786daca47 (patch) | |
tree | 962b975a5c3515804ed5d269019ce0aa3f5da6b3 | |
parent | c3d29f694bf8c910f917be655626d0f80871124f (diff) | |
download | PeerTube-9c6327f803aaf4200672f1fc40b2f43786daca47.tar.gz PeerTube-9c6327f803aaf4200672f1fc40b2f43786daca47.tar.zst PeerTube-9c6327f803aaf4200672f1fc40b2f43786daca47.zip |
Introduce jobs command
-rw-r--r-- | server/tests/api/server/handle-down.ts | 7 | ||||
-rw-r--r-- | server/tests/api/server/jobs.ts | 50 | ||||
-rw-r--r-- | server/tests/api/videos/video-transcoder.ts | 8 | ||||
-rw-r--r-- | shared/extra-utils/index.ts | 1 | ||||
-rw-r--r-- | shared/extra-utils/server/index.ts | 2 | ||||
-rw-r--r-- | shared/extra-utils/server/jobs-command.ts | 35 | ||||
-rw-r--r-- | shared/extra-utils/server/jobs.ts | 67 | ||||
-rw-r--r-- | shared/extra-utils/server/servers.ts | 3 |
8 files changed, 75 insertions, 98 deletions
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index eff4451e5..c6202fdaa 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts | |||
@@ -10,7 +10,6 @@ import { | |||
10 | closeAllSequelize, | 10 | closeAllSequelize, |
11 | completeVideoCheck, | 11 | completeVideoCheck, |
12 | flushAndRunMultipleServers, | 12 | flushAndRunMultipleServers, |
13 | getJobsListPaginationAndSort, | ||
14 | getVideo, | 13 | getVideo, |
15 | getVideoCommentThreads, | 14 | getVideoCommentThreads, |
16 | getVideosList, | 15 | getVideosList, |
@@ -181,15 +180,13 @@ describe('Test handle downs', function () { | |||
181 | const states: JobState[] = [ 'waiting', 'active' ] | 180 | const states: JobState[] = [ 'waiting', 'active' ] |
182 | 181 | ||
183 | for (const state of states) { | 182 | for (const state of states) { |
184 | const res = await getJobsListPaginationAndSort({ | 183 | const body = await servers[0].jobsCommand.getJobsList({ |
185 | url: servers[0].url, | ||
186 | accessToken: servers[0].accessToken, | ||
187 | state: state, | 184 | state: state, |
188 | start: 0, | 185 | start: 0, |
189 | count: 50, | 186 | count: 50, |
190 | sort: '-createdAt' | 187 | sort: '-createdAt' |
191 | }) | 188 | }) |
192 | expect(res.body.data).to.have.length(0) | 189 | expect(body.data).to.have.length(0) |
193 | } | 190 | } |
194 | }) | 191 | }) |
195 | 192 | ||
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts index 6576dd7af..c0b9facff 100644 --- a/server/tests/api/server/jobs.ts +++ b/server/tests/api/server/jobs.ts | |||
@@ -2,13 +2,16 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { cleanupTests, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index' | 5 | import { |
6 | import { dateIsValid } from '../../../../shared/extra-utils/miscs/miscs' | 6 | cleanupTests, |
7 | import { doubleFollow } from '../../../../shared/extra-utils/server/follows' | 7 | dateIsValid, |
8 | import { getJobsList, getJobsListPaginationAndSort, waitJobs } from '../../../../shared/extra-utils/server/jobs' | 8 | doubleFollow, |
9 | import { flushAndRunMultipleServers } from '../../../../shared/extra-utils/server/servers' | 9 | flushAndRunMultipleServers, |
10 | import { uploadVideo } from '../../../../shared/extra-utils/videos/videos' | 10 | ServerInfo, |
11 | import { Job } from '../../../../shared/models/server' | 11 | setAccessTokensToServers, |
12 | uploadVideo, | ||
13 | waitJobs | ||
14 | } from '@shared/extra-utils' | ||
12 | 15 | ||
13 | const expect = chai.expect | 16 | const expect = chai.expect |
14 | 17 | ||
@@ -36,27 +39,25 @@ describe('Test jobs', function () { | |||
36 | }) | 39 | }) |
37 | 40 | ||
38 | it('Should list jobs', async function () { | 41 | it('Should list jobs', async function () { |
39 | const res = await getJobsList(servers[1].url, servers[1].accessToken, 'completed') | 42 | const body = await servers[1].jobsCommand.getJobsList({ state: 'completed' }) |
40 | expect(res.body.total).to.be.above(2) | 43 | expect(body.total).to.be.above(2) |
41 | expect(res.body.data).to.have.length.above(2) | 44 | expect(body.data).to.have.length.above(2) |
42 | }) | 45 | }) |
43 | 46 | ||
44 | it('Should list jobs with sort, pagination and job type', async function () { | 47 | it('Should list jobs with sort, pagination and job type', async function () { |
45 | { | 48 | { |
46 | const res = await getJobsListPaginationAndSort({ | 49 | const body = await servers[1].jobsCommand.getJobsList({ |
47 | url: servers[1].url, | ||
48 | accessToken: servers[1].accessToken, | ||
49 | state: 'completed', | 50 | state: 'completed', |
50 | start: 1, | 51 | start: 1, |
51 | count: 2, | 52 | count: 2, |
52 | sort: 'createdAt' | 53 | sort: 'createdAt' |
53 | }) | 54 | }) |
54 | expect(res.body.total).to.be.above(2) | 55 | expect(body.total).to.be.above(2) |
55 | expect(res.body.data).to.have.lengthOf(2) | 56 | expect(body.data).to.have.lengthOf(2) |
56 | 57 | ||
57 | let job: Job = res.body.data[0] | 58 | let job = body.data[0] |
58 | // Skip repeat jobs | 59 | // Skip repeat jobs |
59 | if (job.type === 'videos-views') job = res.body.data[1] | 60 | if (job.type === 'videos-views') job = body.data[1] |
60 | 61 | ||
61 | expect(job.state).to.equal('completed') | 62 | expect(job.state).to.equal('completed') |
62 | expect(job.type.startsWith('activitypub-')).to.be.true | 63 | expect(job.type.startsWith('activitypub-')).to.be.true |
@@ -66,29 +67,26 @@ describe('Test jobs', function () { | |||
66 | } | 67 | } |
67 | 68 | ||
68 | { | 69 | { |
69 | const res = await getJobsListPaginationAndSort({ | 70 | const body = await servers[1].jobsCommand.getJobsList({ |
70 | url: servers[1].url, | ||
71 | accessToken: servers[1].accessToken, | ||
72 | state: 'completed', | 71 | state: 'completed', |
73 | start: 0, | 72 | start: 0, |
74 | count: 100, | 73 | count: 100, |
75 | sort: 'createdAt', | 74 | sort: 'createdAt', |
76 | jobType: 'activitypub-http-broadcast' | 75 | jobType: 'activitypub-http-broadcast' |
77 | }) | 76 | }) |
78 | expect(res.body.total).to.be.above(2) | 77 | expect(body.total).to.be.above(2) |
79 | 78 | ||
80 | for (const j of res.body.data as Job[]) { | 79 | for (const j of body.data) { |
81 | expect(j.type).to.equal('activitypub-http-broadcast') | 80 | expect(j.type).to.equal('activitypub-http-broadcast') |
82 | } | 81 | } |
83 | } | 82 | } |
84 | }) | 83 | }) |
85 | 84 | ||
86 | it('Should list all jobs', async function () { | 85 | it('Should list all jobs', async function () { |
87 | const res = await getJobsList(servers[1].url, servers[1].accessToken) | 86 | const body = await servers[1].jobsCommand.getJobsList() |
87 | expect(body.total).to.be.above(2) | ||
88 | 88 | ||
89 | const jobs = res.body.data as Job[] | 89 | const jobs = body.data |
90 | |||
91 | expect(res.body.total).to.be.above(2) | ||
92 | expect(jobs).to.have.length.above(2) | 90 | expect(jobs).to.have.length.above(2) |
93 | 91 | ||
94 | // We know there are a least 1 delayed job (video views) and 1 completed job (broadcast) | 92 | // We know there are a least 1 delayed job (video views) and 1 completed job (broadcast) |
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index ea5ffd239..c95053a29 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts | |||
@@ -16,7 +16,6 @@ import { | |||
16 | flushAndRunMultipleServers, | 16 | flushAndRunMultipleServers, |
17 | generateHighBitrateVideo, | 17 | generateHighBitrateVideo, |
18 | generateVideoWithFramerate, | 18 | generateVideoWithFramerate, |
19 | getJobsListPaginationAndSort, | ||
20 | getMyVideos, | 19 | getMyVideos, |
21 | getServerFileSize, | 20 | getServerFileSize, |
22 | getVideo, | 21 | getVideo, |
@@ -709,17 +708,14 @@ describe('Test video transcoding', function () { | |||
709 | describe('Transcoding job queue', function () { | 708 | describe('Transcoding job queue', function () { |
710 | 709 | ||
711 | it('Should have the appropriate priorities for transcoding jobs', async function () { | 710 | it('Should have the appropriate priorities for transcoding jobs', async function () { |
712 | const res = await getJobsListPaginationAndSort({ | 711 | const body = await servers[1].jobsCommand.getJobsList({ |
713 | url: servers[1].url, | ||
714 | accessToken: servers[1].accessToken, | ||
715 | start: 0, | 712 | start: 0, |
716 | count: 100, | 713 | count: 100, |
717 | sort: '-createdAt', | 714 | sort: '-createdAt', |
718 | jobType: 'video-transcoding' | 715 | jobType: 'video-transcoding' |
719 | }) | 716 | }) |
720 | 717 | ||
721 | const jobs = res.body.data as Job[] | 718 | const jobs = body.data |
722 | |||
723 | const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k) | 719 | const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k) |
724 | 720 | ||
725 | expect(transcodingJobs).to.have.lengthOf(14) | 721 | expect(transcodingJobs).to.have.lengthOf(14) |
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts index 48431ed83..652779eea 100644 --- a/shared/extra-utils/index.ts +++ b/shared/extra-utils/index.ts | |||
@@ -15,7 +15,6 @@ export * from './requests/requests' | |||
15 | 15 | ||
16 | export * from './server/clients' | 16 | export * from './server/clients' |
17 | export * from './server/config' | 17 | export * from './server/config' |
18 | export * from './server/jobs' | ||
19 | export * from './server/plugins' | 18 | export * from './server/plugins' |
20 | export * from './server/servers' | 19 | export * from './server/servers' |
21 | 20 | ||
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts index 258084623..b5b6b2116 100644 --- a/shared/extra-utils/server/index.ts +++ b/shared/extra-utils/server/index.ts | |||
@@ -2,3 +2,5 @@ export * from './contact-form-command' | |||
2 | export * from './debug-command' | 2 | export * from './debug-command' |
3 | export * from './follows-command' | 3 | export * from './follows-command' |
4 | export * from './follows' | 4 | export * from './follows' |
5 | export * from './jobs' | ||
6 | export * from './jobs-command' | ||
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts new file mode 100644 index 000000000..758b4a4af --- /dev/null +++ b/shared/extra-utils/server/jobs-command.ts | |||
@@ -0,0 +1,35 @@ | |||
1 | import { pick } from 'lodash' | ||
2 | import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' | ||
3 | import { Job, JobState, JobType, ResultList } from '../../models' | ||
4 | import { AbstractCommand, OverrideCommandOptions } from '../shared' | ||
5 | |||
6 | export class JobsCommand extends AbstractCommand { | ||
7 | |||
8 | getJobsList (options: OverrideCommandOptions & { | ||
9 | state?: JobState | ||
10 | jobType?: JobType | ||
11 | start?: number | ||
12 | count?: number | ||
13 | sort?: string | ||
14 | } = {}) { | ||
15 | const path = this.buildJobsUrl(options.state) | ||
16 | |||
17 | const query = pick(options, [ 'start', 'count', 'sort', 'jobType' ]) | ||
18 | |||
19 | return this.getRequestBody<ResultList<Job>>({ | ||
20 | ...options, | ||
21 | |||
22 | path, | ||
23 | query, | ||
24 | defaultExpectedStatus: HttpStatusCode.OK_200 | ||
25 | }) | ||
26 | } | ||
27 | |||
28 | private buildJobsUrl (state?: JobState) { | ||
29 | let path = '/api/v1/jobs' | ||
30 | |||
31 | if (state) path += '/' + state | ||
32 | |||
33 | return path | ||
34 | } | ||
35 | } | ||
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index a3683913a..b4b3d52e7 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts | |||
@@ -1,57 +1,8 @@ | |||
1 | import * as request from 'supertest' | 1 | |
2 | import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' | 2 | import { JobState } from '../../models' |
3 | import { makeGetRequest } from '../../../shared/extra-utils' | ||
4 | import { Job, JobState, JobType } from '../../models' | ||
5 | import { wait } from '../miscs/miscs' | 3 | import { wait } from '../miscs/miscs' |
6 | import { ServerInfo } from './servers' | 4 | import { ServerInfo } from './servers' |
7 | 5 | ||
8 | function buildJobsUrl (state?: JobState) { | ||
9 | let path = '/api/v1/jobs' | ||
10 | |||
11 | if (state) path += '/' + state | ||
12 | |||
13 | return path | ||
14 | } | ||
15 | |||
16 | function getJobsList (url: string, accessToken: string, state?: JobState) { | ||
17 | const path = buildJobsUrl(state) | ||
18 | |||
19 | return request(url) | ||
20 | .get(path) | ||
21 | .set('Accept', 'application/json') | ||
22 | .set('Authorization', 'Bearer ' + accessToken) | ||
23 | .expect(HttpStatusCode.OK_200) | ||
24 | .expect('Content-Type', /json/) | ||
25 | } | ||
26 | |||
27 | function getJobsListPaginationAndSort (options: { | ||
28 | url: string | ||
29 | accessToken: string | ||
30 | start: number | ||
31 | count: number | ||
32 | sort: string | ||
33 | state?: JobState | ||
34 | jobType?: JobType | ||
35 | }) { | ||
36 | const { url, accessToken, state, start, count, sort, jobType } = options | ||
37 | const path = buildJobsUrl(state) | ||
38 | |||
39 | const query = { | ||
40 | start, | ||
41 | count, | ||
42 | sort, | ||
43 | jobType | ||
44 | } | ||
45 | |||
46 | return makeGetRequest({ | ||
47 | url, | ||
48 | path, | ||
49 | token: accessToken, | ||
50 | statusCodeExpected: HttpStatusCode.OK_200, | ||
51 | query | ||
52 | }) | ||
53 | } | ||
54 | |||
55 | async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { | 6 | async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { |
56 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT | 7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT |
57 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) | 8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) |
@@ -72,15 +23,13 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { | |||
72 | // Check if each server has pending request | 23 | // Check if each server has pending request |
73 | for (const server of servers) { | 24 | for (const server of servers) { |
74 | for (const state of states) { | 25 | for (const state of states) { |
75 | const p = getJobsListPaginationAndSort({ | 26 | const p = server.jobsCommand.getJobsList({ |
76 | url: server.url, | 27 | state, |
77 | accessToken: server.accessToken, | ||
78 | state: state, | ||
79 | start: 0, | 28 | start: 0, |
80 | count: 10, | 29 | count: 10, |
81 | sort: '-createdAt' | 30 | sort: '-createdAt' |
82 | }).then(res => res.body.data) | 31 | }).then(body => body.data) |
83 | .then((jobs: Job[]) => jobs.filter(j => !repeatableJobs.includes(j.type))) | 32 | .then(jobs => jobs.filter(j => !repeatableJobs.includes(j.type))) |
84 | .then(jobs => { | 33 | .then(jobs => { |
85 | if (jobs.length !== 0) { | 34 | if (jobs.length !== 0) { |
86 | pendingRequests = true | 35 | pendingRequests = true |
@@ -122,7 +71,5 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { | |||
122 | // --------------------------------------------------------------------------- | 71 | // --------------------------------------------------------------------------- |
123 | 72 | ||
124 | export { | 73 | export { |
125 | getJobsList, | 74 | waitJobs |
126 | waitJobs, | ||
127 | getJobsListPaginationAndSort | ||
128 | } | 75 | } |
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index 7ac80cea0..5511ce0b0 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts | |||
@@ -19,6 +19,7 @@ import { SearchCommand } from '../search' | |||
19 | import { ContactFormCommand } from './contact-form-command' | 19 | import { ContactFormCommand } from './contact-form-command' |
20 | import { DebugCommand } from './debug-command' | 20 | import { DebugCommand } from './debug-command' |
21 | import { FollowsCommand } from './follows-command' | 21 | import { FollowsCommand } from './follows-command' |
22 | import { JobsCommand } from './jobs-command' | ||
22 | 23 | ||
23 | interface ServerInfo { | 24 | interface ServerInfo { |
24 | app: ChildProcess | 25 | app: ChildProcess |
@@ -83,6 +84,7 @@ interface ServerInfo { | |||
83 | contactFormCommand?: ContactFormCommand | 84 | contactFormCommand?: ContactFormCommand |
84 | debugCommand?: DebugCommand | 85 | debugCommand?: DebugCommand |
85 | followsCommand?: FollowsCommand | 86 | followsCommand?: FollowsCommand |
87 | jobsCommand?: JobsCommand | ||
86 | } | 88 | } |
87 | 89 | ||
88 | function parallelTests () { | 90 | function parallelTests () { |
@@ -299,6 +301,7 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = [] | |||
299 | server.contactFormCommand = new ContactFormCommand(server) | 301 | server.contactFormCommand = new ContactFormCommand(server) |
300 | server.debugCommand = new DebugCommand(server) | 302 | server.debugCommand = new DebugCommand(server) |
301 | server.followsCommand = new FollowsCommand(server) | 303 | server.followsCommand = new FollowsCommand(server) |
304 | server.jobsCommand = new JobsCommand(server) | ||
302 | 305 | ||
303 | res(server) | 306 | res(server) |
304 | }) | 307 | }) |