aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2021-07-07 09:34:56 +0200
committerChocobozzz <me@florianbigard.com>2021-07-20 15:27:17 +0200
commit9c6327f803aaf4200672f1fc40b2f43786daca47 (patch)
tree962b975a5c3515804ed5d269019ce0aa3f5da6b3
parentc3d29f694bf8c910f917be655626d0f80871124f (diff)
downloadPeerTube-9c6327f803aaf4200672f1fc40b2f43786daca47.tar.gz
PeerTube-9c6327f803aaf4200672f1fc40b2f43786daca47.tar.zst
PeerTube-9c6327f803aaf4200672f1fc40b2f43786daca47.zip
Introduce jobs command
-rw-r--r--server/tests/api/server/handle-down.ts7
-rw-r--r--server/tests/api/server/jobs.ts50
-rw-r--r--server/tests/api/videos/video-transcoder.ts8
-rw-r--r--shared/extra-utils/index.ts1
-rw-r--r--shared/extra-utils/server/index.ts2
-rw-r--r--shared/extra-utils/server/jobs-command.ts35
-rw-r--r--shared/extra-utils/server/jobs.ts67
-rw-r--r--shared/extra-utils/server/servers.ts3
8 files changed, 75 insertions, 98 deletions
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index eff4451e5..c6202fdaa 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -10,7 +10,6 @@ import {
10 closeAllSequelize, 10 closeAllSequelize,
11 completeVideoCheck, 11 completeVideoCheck,
12 flushAndRunMultipleServers, 12 flushAndRunMultipleServers,
13 getJobsListPaginationAndSort,
14 getVideo, 13 getVideo,
15 getVideoCommentThreads, 14 getVideoCommentThreads,
16 getVideosList, 15 getVideosList,
@@ -181,15 +180,13 @@ describe('Test handle downs', function () {
181 const states: JobState[] = [ 'waiting', 'active' ] 180 const states: JobState[] = [ 'waiting', 'active' ]
182 181
183 for (const state of states) { 182 for (const state of states) {
184 const res = await getJobsListPaginationAndSort({ 183 const body = await servers[0].jobsCommand.getJobsList({
185 url: servers[0].url,
186 accessToken: servers[0].accessToken,
187 state: state, 184 state: state,
188 start: 0, 185 start: 0,
189 count: 50, 186 count: 50,
190 sort: '-createdAt' 187 sort: '-createdAt'
191 }) 188 })
192 expect(res.body.data).to.have.length(0) 189 expect(body.data).to.have.length(0)
193 } 190 }
194 }) 191 })
195 192
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts
index 6576dd7af..c0b9facff 100644
--- a/server/tests/api/server/jobs.ts
+++ b/server/tests/api/server/jobs.ts
@@ -2,13 +2,16 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { cleanupTests, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index' 5import {
6import { dateIsValid } from '../../../../shared/extra-utils/miscs/miscs' 6 cleanupTests,
7import { doubleFollow } from '../../../../shared/extra-utils/server/follows' 7 dateIsValid,
8import { getJobsList, getJobsListPaginationAndSort, waitJobs } from '../../../../shared/extra-utils/server/jobs' 8 doubleFollow,
9import { flushAndRunMultipleServers } from '../../../../shared/extra-utils/server/servers' 9 flushAndRunMultipleServers,
10import { uploadVideo } from '../../../../shared/extra-utils/videos/videos' 10 ServerInfo,
11import { Job } from '../../../../shared/models/server' 11 setAccessTokensToServers,
12 uploadVideo,
13 waitJobs
14} from '@shared/extra-utils'
12 15
13const expect = chai.expect 16const expect = chai.expect
14 17
@@ -36,27 +39,25 @@ describe('Test jobs', function () {
36 }) 39 })
37 40
38 it('Should list jobs', async function () { 41 it('Should list jobs', async function () {
39 const res = await getJobsList(servers[1].url, servers[1].accessToken, 'completed') 42 const body = await servers[1].jobsCommand.getJobsList({ state: 'completed' })
40 expect(res.body.total).to.be.above(2) 43 expect(body.total).to.be.above(2)
41 expect(res.body.data).to.have.length.above(2) 44 expect(body.data).to.have.length.above(2)
42 }) 45 })
43 46
44 it('Should list jobs with sort, pagination and job type', async function () { 47 it('Should list jobs with sort, pagination and job type', async function () {
45 { 48 {
46 const res = await getJobsListPaginationAndSort({ 49 const body = await servers[1].jobsCommand.getJobsList({
47 url: servers[1].url,
48 accessToken: servers[1].accessToken,
49 state: 'completed', 50 state: 'completed',
50 start: 1, 51 start: 1,
51 count: 2, 52 count: 2,
52 sort: 'createdAt' 53 sort: 'createdAt'
53 }) 54 })
54 expect(res.body.total).to.be.above(2) 55 expect(body.total).to.be.above(2)
55 expect(res.body.data).to.have.lengthOf(2) 56 expect(body.data).to.have.lengthOf(2)
56 57
57 let job: Job = res.body.data[0] 58 let job = body.data[0]
58 // Skip repeat jobs 59 // Skip repeat jobs
59 if (job.type === 'videos-views') job = res.body.data[1] 60 if (job.type === 'videos-views') job = body.data[1]
60 61
61 expect(job.state).to.equal('completed') 62 expect(job.state).to.equal('completed')
62 expect(job.type.startsWith('activitypub-')).to.be.true 63 expect(job.type.startsWith('activitypub-')).to.be.true
@@ -66,29 +67,26 @@ describe('Test jobs', function () {
66 } 67 }
67 68
68 { 69 {
69 const res = await getJobsListPaginationAndSort({ 70 const body = await servers[1].jobsCommand.getJobsList({
70 url: servers[1].url,
71 accessToken: servers[1].accessToken,
72 state: 'completed', 71 state: 'completed',
73 start: 0, 72 start: 0,
74 count: 100, 73 count: 100,
75 sort: 'createdAt', 74 sort: 'createdAt',
76 jobType: 'activitypub-http-broadcast' 75 jobType: 'activitypub-http-broadcast'
77 }) 76 })
78 expect(res.body.total).to.be.above(2) 77 expect(body.total).to.be.above(2)
79 78
80 for (const j of res.body.data as Job[]) { 79 for (const j of body.data) {
81 expect(j.type).to.equal('activitypub-http-broadcast') 80 expect(j.type).to.equal('activitypub-http-broadcast')
82 } 81 }
83 } 82 }
84 }) 83 })
85 84
86 it('Should list all jobs', async function () { 85 it('Should list all jobs', async function () {
87 const res = await getJobsList(servers[1].url, servers[1].accessToken) 86 const body = await servers[1].jobsCommand.getJobsList()
87 expect(body.total).to.be.above(2)
88 88
89 const jobs = res.body.data as Job[] 89 const jobs = body.data
90
91 expect(res.body.total).to.be.above(2)
92 expect(jobs).to.have.length.above(2) 90 expect(jobs).to.have.length.above(2)
93 91
94 // We know there are a least 1 delayed job (video views) and 1 completed job (broadcast) 92 // We know there are a least 1 delayed job (video views) and 1 completed job (broadcast)
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index ea5ffd239..c95053a29 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -16,7 +16,6 @@ import {
16 flushAndRunMultipleServers, 16 flushAndRunMultipleServers,
17 generateHighBitrateVideo, 17 generateHighBitrateVideo,
18 generateVideoWithFramerate, 18 generateVideoWithFramerate,
19 getJobsListPaginationAndSort,
20 getMyVideos, 19 getMyVideos,
21 getServerFileSize, 20 getServerFileSize,
22 getVideo, 21 getVideo,
@@ -709,17 +708,14 @@ describe('Test video transcoding', function () {
709 describe('Transcoding job queue', function () { 708 describe('Transcoding job queue', function () {
710 709
711 it('Should have the appropriate priorities for transcoding jobs', async function () { 710 it('Should have the appropriate priorities for transcoding jobs', async function () {
712 const res = await getJobsListPaginationAndSort({ 711 const body = await servers[1].jobsCommand.getJobsList({
713 url: servers[1].url,
714 accessToken: servers[1].accessToken,
715 start: 0, 712 start: 0,
716 count: 100, 713 count: 100,
717 sort: '-createdAt', 714 sort: '-createdAt',
718 jobType: 'video-transcoding' 715 jobType: 'video-transcoding'
719 }) 716 })
720 717
721 const jobs = res.body.data as Job[] 718 const jobs = body.data
722
723 const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k) 719 const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k)
724 720
725 expect(transcodingJobs).to.have.lengthOf(14) 721 expect(transcodingJobs).to.have.lengthOf(14)
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts
index 48431ed83..652779eea 100644
--- a/shared/extra-utils/index.ts
+++ b/shared/extra-utils/index.ts
@@ -15,7 +15,6 @@ export * from './requests/requests'
15 15
16export * from './server/clients' 16export * from './server/clients'
17export * from './server/config' 17export * from './server/config'
18export * from './server/jobs'
19export * from './server/plugins' 18export * from './server/plugins'
20export * from './server/servers' 19export * from './server/servers'
21 20
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts
index 258084623..b5b6b2116 100644
--- a/shared/extra-utils/server/index.ts
+++ b/shared/extra-utils/server/index.ts
@@ -2,3 +2,5 @@ export * from './contact-form-command'
2export * from './debug-command' 2export * from './debug-command'
3export * from './follows-command' 3export * from './follows-command'
4export * from './follows' 4export * from './follows'
5export * from './jobs'
6export * from './jobs-command'
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts
new file mode 100644
index 000000000..758b4a4af
--- /dev/null
+++ b/shared/extra-utils/server/jobs-command.ts
@@ -0,0 +1,35 @@
1import { pick } from 'lodash'
2import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes'
3import { Job, JobState, JobType, ResultList } from '../../models'
4import { AbstractCommand, OverrideCommandOptions } from '../shared'
5
6export class JobsCommand extends AbstractCommand {
7
8 getJobsList (options: OverrideCommandOptions & {
9 state?: JobState
10 jobType?: JobType
11 start?: number
12 count?: number
13 sort?: string
14 } = {}) {
15 const path = this.buildJobsUrl(options.state)
16
17 const query = pick(options, [ 'start', 'count', 'sort', 'jobType' ])
18
19 return this.getRequestBody<ResultList<Job>>({
20 ...options,
21
22 path,
23 query,
24 defaultExpectedStatus: HttpStatusCode.OK_200
25 })
26 }
27
28 private buildJobsUrl (state?: JobState) {
29 let path = '/api/v1/jobs'
30
31 if (state) path += '/' + state
32
33 return path
34 }
35}
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts
index a3683913a..b4b3d52e7 100644
--- a/shared/extra-utils/server/jobs.ts
+++ b/shared/extra-utils/server/jobs.ts
@@ -1,57 +1,8 @@
1import * as request from 'supertest' 1
2import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' 2import { JobState } from '../../models'
3import { makeGetRequest } from '../../../shared/extra-utils'
4import { Job, JobState, JobType } from '../../models'
5import { wait } from '../miscs/miscs' 3import { wait } from '../miscs/miscs'
6import { ServerInfo } from './servers' 4import { ServerInfo } from './servers'
7 5
8function buildJobsUrl (state?: JobState) {
9 let path = '/api/v1/jobs'
10
11 if (state) path += '/' + state
12
13 return path
14}
15
16function getJobsList (url: string, accessToken: string, state?: JobState) {
17 const path = buildJobsUrl(state)
18
19 return request(url)
20 .get(path)
21 .set('Accept', 'application/json')
22 .set('Authorization', 'Bearer ' + accessToken)
23 .expect(HttpStatusCode.OK_200)
24 .expect('Content-Type', /json/)
25}
26
27function getJobsListPaginationAndSort (options: {
28 url: string
29 accessToken: string
30 start: number
31 count: number
32 sort: string
33 state?: JobState
34 jobType?: JobType
35}) {
36 const { url, accessToken, state, start, count, sort, jobType } = options
37 const path = buildJobsUrl(state)
38
39 const query = {
40 start,
41 count,
42 sort,
43 jobType
44 }
45
46 return makeGetRequest({
47 url,
48 path,
49 token: accessToken,
50 statusCodeExpected: HttpStatusCode.OK_200,
51 query
52 })
53}
54
55async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { 6async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
56 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT 7 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT
57 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) 8 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10)
@@ -72,15 +23,13 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
72 // Check if each server has pending request 23 // Check if each server has pending request
73 for (const server of servers) { 24 for (const server of servers) {
74 for (const state of states) { 25 for (const state of states) {
75 const p = getJobsListPaginationAndSort({ 26 const p = server.jobsCommand.getJobsList({
76 url: server.url, 27 state,
77 accessToken: server.accessToken,
78 state: state,
79 start: 0, 28 start: 0,
80 count: 10, 29 count: 10,
81 sort: '-createdAt' 30 sort: '-createdAt'
82 }).then(res => res.body.data) 31 }).then(body => body.data)
83 .then((jobs: Job[]) => jobs.filter(j => !repeatableJobs.includes(j.type))) 32 .then(jobs => jobs.filter(j => !repeatableJobs.includes(j.type)))
84 .then(jobs => { 33 .then(jobs => {
85 if (jobs.length !== 0) { 34 if (jobs.length !== 0) {
86 pendingRequests = true 35 pendingRequests = true
@@ -122,7 +71,5 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
122// --------------------------------------------------------------------------- 71// ---------------------------------------------------------------------------
123 72
124export { 73export {
125 getJobsList, 74 waitJobs
126 waitJobs,
127 getJobsListPaginationAndSort
128} 75}
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts
index 7ac80cea0..5511ce0b0 100644
--- a/shared/extra-utils/server/servers.ts
+++ b/shared/extra-utils/server/servers.ts
@@ -19,6 +19,7 @@ import { SearchCommand } from '../search'
19import { ContactFormCommand } from './contact-form-command' 19import { ContactFormCommand } from './contact-form-command'
20import { DebugCommand } from './debug-command' 20import { DebugCommand } from './debug-command'
21import { FollowsCommand } from './follows-command' 21import { FollowsCommand } from './follows-command'
22import { JobsCommand } from './jobs-command'
22 23
23interface ServerInfo { 24interface ServerInfo {
24 app: ChildProcess 25 app: ChildProcess
@@ -83,6 +84,7 @@ interface ServerInfo {
83 contactFormCommand?: ContactFormCommand 84 contactFormCommand?: ContactFormCommand
84 debugCommand?: DebugCommand 85 debugCommand?: DebugCommand
85 followsCommand?: FollowsCommand 86 followsCommand?: FollowsCommand
87 jobsCommand?: JobsCommand
86} 88}
87 89
88function parallelTests () { 90function parallelTests () {
@@ -299,6 +301,7 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = []
299 server.contactFormCommand = new ContactFormCommand(server) 301 server.contactFormCommand = new ContactFormCommand(server)
300 server.debugCommand = new DebugCommand(server) 302 server.debugCommand = new DebugCommand(server)
301 server.followsCommand = new FollowsCommand(server) 303 server.followsCommand = new FollowsCommand(server)
304 server.jobsCommand = new JobsCommand(server)
302 305
303 res(server) 306 res(server)
304 }) 307 })