aboutsummaryrefslogtreecommitdiffhomepage
path: root/shared/server-commands
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-04-21 15:00:01 +0200
committerChocobozzz <chocobozzz@cpy.re>2023-05-09 08:57:34 +0200
commitd102de1b38f2877463529c3b27bd35ffef4fd8bf (patch)
tree31fa0bdf26ad7a2ee46d600d804a6f03260266c8 /shared/server-commands
parent2fe978744e5b74eb824e4d79c1bb9b840169f125 (diff)
downloadPeerTube-d102de1b38f2877463529c3b27bd35ffef4fd8bf.tar.gz
PeerTube-d102de1b38f2877463529c3b27bd35ffef4fd8bf.tar.zst
PeerTube-d102de1b38f2877463529c3b27bd35ffef4fd8bf.zip
Add runner server tests
Diffstat (limited to 'shared/server-commands')
-rw-r--r--shared/server-commands/index.ts2
-rw-r--r--shared/server-commands/miscs/index.ts2
-rw-r--r--shared/server-commands/miscs/sql-command.ts146
-rw-r--r--shared/server-commands/miscs/webtorrent.ts46
-rw-r--r--shared/server-commands/requests/requests.ts37
-rw-r--r--shared/server-commands/runners/index.ts3
-rw-r--r--shared/server-commands/runners/runner-jobs-command.ts279
-rw-r--r--shared/server-commands/runners/runner-registration-tokens-command.ts55
-rw-r--r--shared/server-commands/runners/runners-command.ts77
-rw-r--r--shared/server-commands/server/config-command.ts34
-rw-r--r--shared/server-commands/server/jobs.ts26
-rw-r--r--shared/server-commands/server/server.ts20
-rw-r--r--shared/server-commands/server/servers.ts2
-rw-r--r--shared/server-commands/shared/abstract-command.ts4
-rw-r--r--shared/server-commands/socket/socket-io-command.ts9
-rw-r--r--shared/server-commands/videos/live-command.ts2
-rw-r--r--shared/server-commands/videos/streaming-playlists-command.ts4
17 files changed, 527 insertions, 221 deletions
diff --git a/shared/server-commands/index.ts b/shared/server-commands/index.ts
index c24ebb2df..a4581dbc0 100644
--- a/shared/server-commands/index.ts
+++ b/shared/server-commands/index.ts
@@ -3,10 +3,10 @@ export * from './cli'
3export * from './custom-pages' 3export * from './custom-pages'
4export * from './feeds' 4export * from './feeds'
5export * from './logs' 5export * from './logs'
6export * from './miscs'
7export * from './moderation' 6export * from './moderation'
8export * from './overviews' 7export * from './overviews'
9export * from './requests' 8export * from './requests'
9export * from './runners'
10export * from './search' 10export * from './search'
11export * from './server' 11export * from './server'
12export * from './socket' 12export * from './socket'
diff --git a/shared/server-commands/miscs/index.ts b/shared/server-commands/miscs/index.ts
deleted file mode 100644
index a1d14e998..000000000
--- a/shared/server-commands/miscs/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
1export * from './sql-command'
2export * from './webtorrent'
diff --git a/shared/server-commands/miscs/sql-command.ts b/shared/server-commands/miscs/sql-command.ts
deleted file mode 100644
index 35cc2253f..000000000
--- a/shared/server-commands/miscs/sql-command.ts
+++ /dev/null
@@ -1,146 +0,0 @@
1import { QueryTypes, Sequelize } from 'sequelize'
2import { forceNumber } from '@shared/core-utils'
3import { AbstractCommand } from '../shared'
4
5export class SQLCommand extends AbstractCommand {
6 private sequelize: Sequelize
7
8 deleteAll (table: string) {
9 const seq = this.getSequelize()
10
11 const options = { type: QueryTypes.DELETE }
12
13 return seq.query(`DELETE FROM "${table}"`, options)
14 }
15
16 async getVideoShareCount () {
17 const [ { total } ] = await this.selectQuery<{ total: string }>(`SELECT COUNT(*) as total FROM "videoShare"`)
18 if (total === null) return 0
19
20 return parseInt(total, 10)
21 }
22
23 async getInternalFileUrl (fileId: number) {
24 return this.selectQuery<{ fileUrl: string }>(`SELECT "fileUrl" FROM "videoFile" WHERE id = :fileId`, { fileId })
25 .then(rows => rows[0].fileUrl)
26 }
27
28 setActorField (to: string, field: string, value: string) {
29 return this.updateQuery(`UPDATE actor SET ${this.escapeColumnName(field)} = :value WHERE url = :to`, { value, to })
30 }
31
32 setVideoField (uuid: string, field: string, value: string) {
33 return this.updateQuery(`UPDATE video SET ${this.escapeColumnName(field)} = :value WHERE uuid = :uuid`, { value, uuid })
34 }
35
36 setPlaylistField (uuid: string, field: string, value: string) {
37 return this.updateQuery(`UPDATE "videoPlaylist" SET ${this.escapeColumnName(field)} = :value WHERE uuid = :uuid`, { value, uuid })
38 }
39
40 async countVideoViewsOf (uuid: string) {
41 const query = 'SELECT SUM("videoView"."views") AS "total" FROM "videoView" ' +
42 `INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = :uuid`
43
44 const [ { total } ] = await this.selectQuery<{ total: number }>(query, { uuid })
45 if (!total) return 0
46
47 return forceNumber(total)
48 }
49
50 getActorImage (filename: string) {
51 return this.selectQuery<{ width: number, height: number }>(`SELECT * FROM "actorImage" WHERE filename = :filename`, { filename })
52 .then(rows => rows[0])
53 }
54
55 // ---------------------------------------------------------------------------
56
57 setPluginVersion (pluginName: string, newVersion: string) {
58 return this.setPluginField(pluginName, 'version', newVersion)
59 }
60
61 setPluginLatestVersion (pluginName: string, newVersion: string) {
62 return this.setPluginField(pluginName, 'latestVersion', newVersion)
63 }
64
65 setPluginField (pluginName: string, field: string, value: string) {
66 return this.updateQuery(
67 `UPDATE "plugin" SET ${this.escapeColumnName(field)} = :value WHERE "name" = :pluginName`,
68 { pluginName, value }
69 )
70 }
71
72 // ---------------------------------------------------------------------------
73
74 selectQuery <T extends object> (query: string, replacements: { [id: string]: string | number } = {}) {
75 const seq = this.getSequelize()
76 const options = {
77 type: QueryTypes.SELECT as QueryTypes.SELECT,
78 replacements
79 }
80
81 return seq.query<T>(query, options)
82 }
83
84 updateQuery (query: string, replacements: { [id: string]: string | number } = {}) {
85 const seq = this.getSequelize()
86 const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, replacements }
87
88 return seq.query(query, options)
89 }
90
91 // ---------------------------------------------------------------------------
92
93 async getPlaylistInfohash (playlistId: number) {
94 const query = 'SELECT "p2pMediaLoaderInfohashes" FROM "videoStreamingPlaylist" WHERE id = :playlistId'
95
96 const result = await this.selectQuery<{ p2pMediaLoaderInfohashes: string }>(query, { playlistId })
97 if (!result || result.length === 0) return []
98
99 return result[0].p2pMediaLoaderInfohashes
100 }
101
102 // ---------------------------------------------------------------------------
103
104 setActorFollowScores (newScore: number) {
105 return this.updateQuery(`UPDATE "actorFollow" SET "score" = :newScore`, { newScore })
106 }
107
108 setTokenField (accessToken: string, field: string, value: string) {
109 return this.updateQuery(
110 `UPDATE "oAuthToken" SET ${this.escapeColumnName(field)} = :value WHERE "accessToken" = :accessToken`,
111 { value, accessToken }
112 )
113 }
114
115 async cleanup () {
116 if (!this.sequelize) return
117
118 await this.sequelize.close()
119 this.sequelize = undefined
120 }
121
122 private getSequelize () {
123 if (this.sequelize) return this.sequelize
124
125 const dbname = 'peertube_test' + this.server.internalServerNumber
126 const username = 'peertube'
127 const password = 'peertube'
128 const host = '127.0.0.1'
129 const port = 5432
130
131 this.sequelize = new Sequelize(dbname, username, password, {
132 dialect: 'postgres',
133 host,
134 port,
135 logging: false
136 })
137
138 return this.sequelize
139 }
140
141 private escapeColumnName (columnName: string) {
142 return this.getSequelize().escape(columnName)
143 .replace(/^'/, '"')
144 .replace(/'$/, '"')
145 }
146}
diff --git a/shared/server-commands/miscs/webtorrent.ts b/shared/server-commands/miscs/webtorrent.ts
deleted file mode 100644
index 0683f8893..000000000
--- a/shared/server-commands/miscs/webtorrent.ts
+++ /dev/null
@@ -1,46 +0,0 @@
1import { readFile } from 'fs-extra'
2import parseTorrent from 'parse-torrent'
3import { basename, join } from 'path'
4import * as WebTorrent from 'webtorrent'
5import { VideoFile } from '@shared/models'
6import { PeerTubeServer } from '../server'
7
8let webtorrent: WebTorrent.Instance
9
10function webtorrentAdd (torrentId: string, refreshWebTorrent = false) {
11 const WebTorrent = require('webtorrent')
12
13 if (webtorrent && refreshWebTorrent) webtorrent.destroy()
14 if (!webtorrent || refreshWebTorrent) webtorrent = new WebTorrent()
15
16 webtorrent.on('error', err => console.error('Error in webtorrent', err))
17
18 return new Promise<WebTorrent.Torrent>(res => {
19 const torrent = webtorrent.add(torrentId, res)
20
21 torrent.on('error', err => console.error('Error in webtorrent torrent', err))
22 torrent.on('warning', warn => {
23 const msg = typeof warn === 'string'
24 ? warn
25 : warn.message
26
27 if (msg.includes('Unsupported')) return
28
29 console.error('Warning in webtorrent torrent', warn)
30 })
31 })
32}
33
34async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) {
35 const torrentName = basename(file.torrentUrl)
36 const torrentPath = server.servers.buildDirectory(join('torrents', torrentName))
37
38 const data = await readFile(torrentPath)
39
40 return parseTorrent(data)
41}
42
43export {
44 webtorrentAdd,
45 parseTorrentVideo
46}
diff --git a/shared/server-commands/requests/requests.ts b/shared/server-commands/requests/requests.ts
index cb0e1a5fb..96f67b4c7 100644
--- a/shared/server-commands/requests/requests.ts
+++ b/shared/server-commands/requests/requests.ts
@@ -10,6 +10,7 @@ export type CommonRequestParams = {
10 url: string 10 url: string
11 path?: string 11 path?: string
12 contentType?: string 12 contentType?: string
13 responseType?: string
13 range?: string 14 range?: string
14 redirects?: number 15 redirects?: number
15 accept?: string 16 accept?: string
@@ -27,16 +28,23 @@ function makeRawRequest (options: {
27 expectedStatus?: HttpStatusCode 28 expectedStatus?: HttpStatusCode
28 range?: string 29 range?: string
29 query?: { [ id: string ]: string } 30 query?: { [ id: string ]: string }
31 method?: 'GET' | 'POST'
30}) { 32}) {
31 const { host, protocol, pathname } = new URL(options.url) 33 const { host, protocol, pathname } = new URL(options.url)
32 34
33 return makeGetRequest({ 35 const reqOptions = {
34 url: `${protocol}//${host}`, 36 url: `${protocol}//${host}`,
35 path: pathname, 37 path: pathname,
36 contentType: undefined, 38 contentType: undefined,
37 39
38 ...pick(options, [ 'expectedStatus', 'range', 'token', 'query' ]) 40 ...pick(options, [ 'expectedStatus', 'range', 'token', 'query' ])
39 }) 41 }
42
43 if (options.method === 'POST') {
44 return makePostBodyRequest(reqOptions)
45 }
46
47 return makeGetRequest(reqOptions)
40} 48}
41 49
42function makeGetRequest (options: CommonRequestParams & { 50function makeGetRequest (options: CommonRequestParams & {
@@ -135,6 +143,8 @@ function decodeQueryString (path: string) {
135 return decode(path.split('?')[1]) 143 return decode(path.split('?')[1])
136} 144}
137 145
146// ---------------------------------------------------------------------------
147
138function unwrapBody <T> (test: request.Test): Promise<T> { 148function unwrapBody <T> (test: request.Test): Promise<T> {
139 return test.then(res => res.body) 149 return test.then(res => res.body)
140} 150}
@@ -149,7 +159,16 @@ function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> {
149 try { 159 try {
150 return JSON.parse(new TextDecoder().decode(res.body)) 160 return JSON.parse(new TextDecoder().decode(res.body))
151 } catch (err) { 161 } catch (err) {
152 console.error('Cannot decode JSON.', res.body) 162 console.error('Cannot decode JSON.', res.body instanceof Buffer ? res.body.toString() : res.body)
163 throw err
164 }
165 }
166
167 if (res.text) {
168 try {
169 return JSON.parse(res.text)
170 } catch (err) {
171 console.error('Cannot decode json', res.text)
153 throw err 172 throw err
154 } 173 }
155 } 174 }
@@ -184,6 +203,7 @@ export {
184 203
185function buildRequest (req: request.Test, options: CommonRequestParams) { 204function buildRequest (req: request.Test, options: CommonRequestParams) {
186 if (options.contentType) req.set('Accept', options.contentType) 205 if (options.contentType) req.set('Accept', options.contentType)
206 if (options.responseType) req.responseType(options.responseType)
187 if (options.token) req.set('Authorization', 'Bearer ' + options.token) 207 if (options.token) req.set('Authorization', 'Bearer ' + options.token)
188 if (options.range) req.set('Range', options.range) 208 if (options.range) req.set('Range', options.range)
189 if (options.accept) req.set('Accept', options.accept) 209 if (options.accept) req.set('Accept', options.accept)
@@ -196,13 +216,18 @@ function buildRequest (req: request.Test, options: CommonRequestParams) {
196 req.set(name, options.headers[name]) 216 req.set(name, options.headers[name])
197 }) 217 })
198 218
199 return req.expect((res) => { 219 return req.expect(res => {
200 if (options.expectedStatus && res.status !== options.expectedStatus) { 220 if (options.expectedStatus && res.status !== options.expectedStatus) {
201 throw new Error(`Expected status ${options.expectedStatus}, got ${res.status}. ` + 221 const err = new Error(`Expected status ${options.expectedStatus}, got ${res.status}. ` +
202 `\nThe server responded: "${res.body?.error ?? res.text}".\n` + 222 `\nThe server responded: "${res.body?.error ?? res.text}".\n` +
203 'You may take a closer look at the logs. To see how to do so, check out this page: ' + 223 'You may take a closer look at the logs. To see how to do so, check out this page: ' +
204 'https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/development/tests.md#debug-server-logs') 224 'https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/development/tests.md#debug-server-logs');
225
226 (err as any).res = res
227
228 throw err
205 } 229 }
230
206 return res 231 return res
207 }) 232 })
208} 233}
diff --git a/shared/server-commands/runners/index.ts b/shared/server-commands/runners/index.ts
new file mode 100644
index 000000000..9e8e1baf2
--- /dev/null
+++ b/shared/server-commands/runners/index.ts
@@ -0,0 +1,3 @@
1export * from './runner-jobs-command'
2export * from './runner-registration-tokens-command'
3export * from './runners-command'
diff --git a/shared/server-commands/runners/runner-jobs-command.ts b/shared/server-commands/runners/runner-jobs-command.ts
new file mode 100644
index 000000000..3b0f84b9d
--- /dev/null
+++ b/shared/server-commands/runners/runner-jobs-command.ts
@@ -0,0 +1,279 @@
1import { omit, pick, wait } from '@shared/core-utils'
2import {
3 AbortRunnerJobBody,
4 AcceptRunnerJobBody,
5 AcceptRunnerJobResult,
6 ErrorRunnerJobBody,
7 HttpStatusCode,
8 isHLSTranscodingPayloadSuccess,
9 isLiveRTMPHLSTranscodingUpdatePayload,
10 isWebVideoOrAudioMergeTranscodingPayloadSuccess,
11 RequestRunnerJobBody,
12 RequestRunnerJobResult,
13 ResultList,
14 RunnerJobAdmin,
15 RunnerJobLiveRTMPHLSTranscodingPayload,
16 RunnerJobPayload,
17 RunnerJobState,
18 RunnerJobSuccessBody,
19 RunnerJobSuccessPayload,
20 RunnerJobType,
21 RunnerJobUpdateBody,
22 RunnerJobVODPayload
23} from '@shared/models'
24import { unwrapBody } from '../requests'
25import { waitJobs } from '../server'
26import { AbstractCommand, OverrideCommandOptions } from '../shared'
27
28export class RunnerJobsCommand extends AbstractCommand {
29
30 list (options: OverrideCommandOptions & {
31 start?: number
32 count?: number
33 sort?: string
34 search?: string
35 } = {}) {
36 const path = '/api/v1/runners/jobs'
37
38 return this.getRequestBody<ResultList<RunnerJobAdmin>>({
39 ...options,
40
41 path,
42 query: pick(options, [ 'start', 'count', 'sort', 'search' ]),
43 implicitToken: true,
44 defaultExpectedStatus: HttpStatusCode.OK_200
45 })
46 }
47
48 cancelByAdmin (options: OverrideCommandOptions & { jobUUID: string }) {
49 const path = '/api/v1/runners/jobs/' + options.jobUUID + '/cancel'
50
51 return this.postBodyRequest({
52 ...options,
53
54 path,
55 implicitToken: true,
56 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
57 })
58 }
59
60 // ---------------------------------------------------------------------------
61
62 request (options: OverrideCommandOptions & RequestRunnerJobBody) {
63 const path = '/api/v1/runners/jobs/request'
64
65 return unwrapBody<RequestRunnerJobResult>(this.postBodyRequest({
66 ...options,
67
68 path,
69 fields: pick(options, [ 'runnerToken' ]),
70 implicitToken: false,
71 defaultExpectedStatus: HttpStatusCode.OK_200
72 }))
73 }
74
75 async requestVOD (options: OverrideCommandOptions & RequestRunnerJobBody) {
76 const vodTypes = new Set<RunnerJobType>([ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ])
77
78 const { availableJobs } = await this.request(options)
79
80 return {
81 availableJobs: availableJobs.filter(j => vodTypes.has(j.type))
82 } as RequestRunnerJobResult<RunnerJobVODPayload>
83 }
84
85 async requestLive (options: OverrideCommandOptions & RequestRunnerJobBody) {
86 const vodTypes = new Set<RunnerJobType>([ 'live-rtmp-hls-transcoding' ])
87
88 const { availableJobs } = await this.request(options)
89
90 return {
91 availableJobs: availableJobs.filter(j => vodTypes.has(j.type))
92 } as RequestRunnerJobResult<RunnerJobLiveRTMPHLSTranscodingPayload>
93 }
94
95 // ---------------------------------------------------------------------------
96
97 accept <T extends RunnerJobPayload = RunnerJobPayload> (options: OverrideCommandOptions & AcceptRunnerJobBody & { jobUUID: string }) {
98 const path = '/api/v1/runners/jobs/' + options.jobUUID + '/accept'
99
100 return unwrapBody<AcceptRunnerJobResult<T>>(this.postBodyRequest({
101 ...options,
102
103 path,
104 fields: pick(options, [ 'runnerToken' ]),
105 implicitToken: false,
106 defaultExpectedStatus: HttpStatusCode.OK_200
107 }))
108 }
109
110 abort (options: OverrideCommandOptions & AbortRunnerJobBody & { jobUUID: string }) {
111 const path = '/api/v1/runners/jobs/' + options.jobUUID + '/abort'
112
113 return this.postBodyRequest({
114 ...options,
115
116 path,
117 fields: pick(options, [ 'reason', 'jobToken', 'runnerToken' ]),
118 implicitToken: false,
119 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
120 })
121 }
122
123 update (options: OverrideCommandOptions & RunnerJobUpdateBody & { jobUUID: string }) {
124 const path = '/api/v1/runners/jobs/' + options.jobUUID + '/update'
125
126 const { payload } = options
127 const attaches: { [id: string]: any } = {}
128 let payloadWithoutFiles = payload
129
130 if (isLiveRTMPHLSTranscodingUpdatePayload(payload)) {
131 if (payload.masterPlaylistFile) {
132 attaches[`payload[masterPlaylistFile]`] = payload.masterPlaylistFile
133 }
134
135 attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile
136 attaches[`payload[videoChunkFile]`] = payload.videoChunkFile
137
138 payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'masterPlaylistFile', 'resolutionPlaylistFile', 'videoChunkFile' ])
139 }
140
141 return this.postUploadRequest({
142 ...options,
143
144 path,
145 fields: {
146 ...pick(options, [ 'progress', 'jobToken', 'runnerToken' ]),
147
148 payload: payloadWithoutFiles
149 },
150 attaches,
151 implicitToken: false,
152 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
153 })
154 }
155
156 error (options: OverrideCommandOptions & ErrorRunnerJobBody & { jobUUID: string }) {
157 const path = '/api/v1/runners/jobs/' + options.jobUUID + '/error'
158
159 return this.postBodyRequest({
160 ...options,
161
162 path,
163 fields: pick(options, [ 'message', 'jobToken', 'runnerToken' ]),
164 implicitToken: false,
165 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
166 })
167 }
168
169 success (options: OverrideCommandOptions & RunnerJobSuccessBody & { jobUUID: string }) {
170 const { payload } = options
171
172 const path = '/api/v1/runners/jobs/' + options.jobUUID + '/success'
173 const attaches: { [id: string]: any } = {}
174 let payloadWithoutFiles = payload
175
176 if ((isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) && payload.videoFile) {
177 attaches[`payload[videoFile]`] = payload.videoFile
178
179 payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'videoFile' ])
180 }
181
182 if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile) {
183 attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile
184
185 payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'resolutionPlaylistFile' ])
186 }
187
188 return this.postUploadRequest({
189 ...options,
190
191 path,
192 attaches,
193 fields: {
194 ...pick(options, [ 'jobToken', 'runnerToken' ]),
195
196 payload: payloadWithoutFiles
197 },
198 implicitToken: false,
199 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
200 })
201 }
202
203 getInputFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) {
204 const { host, protocol, pathname } = new URL(options.url)
205
206 return this.postBodyRequest({
207 url: `${protocol}//${host}`,
208 path: pathname,
209
210 fields: pick(options, [ 'jobToken', 'runnerToken' ]),
211 implicitToken: false,
212 defaultExpectedStatus: HttpStatusCode.OK_200
213 })
214 }
215
216 // ---------------------------------------------------------------------------
217
218 async autoAccept (options: OverrideCommandOptions & RequestRunnerJobBody & { type?: RunnerJobType }) {
219 const { availableJobs } = await this.request(options)
220
221 const job = options.type
222 ? availableJobs.find(j => j.type === options.type)
223 : availableJobs[0]
224
225 return this.accept({ ...options, jobUUID: job.uuid })
226 }
227
228 async autoProcessWebVideoJob (runnerToken: string, jobUUIDToProcess?: string) {
229 let jobUUID = jobUUIDToProcess
230
231 if (!jobUUID) {
232 const { availableJobs } = await this.request({ runnerToken })
233 jobUUID = availableJobs[0].uuid
234 }
235
236 const { job } = await this.accept({ runnerToken, jobUUID })
237 const jobToken = job.jobToken
238
239 const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' }
240 await this.success({ runnerToken, jobUUID, jobToken, payload })
241
242 await waitJobs([ this.server ])
243
244 return job
245 }
246
247 async cancelAllJobs (options: { state?: RunnerJobState } = {}) {
248 const { state } = options
249
250 const { data } = await this.list({ count: 100 })
251
252 for (const job of data) {
253 if (state && job.state.id !== state) continue
254
255 await this.cancelByAdmin({ jobUUID: job.uuid })
256 }
257 }
258
259 async getJob (options: OverrideCommandOptions & { uuid: string }) {
260 const { data } = await this.list({ ...options, count: 100, sort: '-updatedAt' })
261
262 return data.find(j => j.uuid === options.uuid)
263 }
264
265 async requestLiveJob (runnerToken: string) {
266 let availableJobs: RequestRunnerJobResult<RunnerJobLiveRTMPHLSTranscodingPayload>['availableJobs'] = []
267
268 while (availableJobs.length === 0) {
269 const result = await this.requestLive({ runnerToken })
270 availableJobs = result.availableJobs
271
272 if (availableJobs.length === 1) break
273
274 await wait(150)
275 }
276
277 return availableJobs[0]
278 }
279}
diff --git a/shared/server-commands/runners/runner-registration-tokens-command.ts b/shared/server-commands/runners/runner-registration-tokens-command.ts
new file mode 100644
index 000000000..e4f2e3d95
--- /dev/null
+++ b/shared/server-commands/runners/runner-registration-tokens-command.ts
@@ -0,0 +1,55 @@
1import { pick } from '@shared/core-utils'
2import { HttpStatusCode, ResultList, RunnerRegistrationToken } from '@shared/models'
3import { AbstractCommand, OverrideCommandOptions } from '../shared'
4
5export class RunnerRegistrationTokensCommand extends AbstractCommand {
6
7 list (options: OverrideCommandOptions & {
8 start?: number
9 count?: number
10 sort?: string
11 } = {}) {
12 const path = '/api/v1/runners/registration-tokens'
13
14 return this.getRequestBody<ResultList<RunnerRegistrationToken>>({
15 ...options,
16
17 path,
18 query: pick(options, [ 'start', 'count', 'sort' ]),
19 implicitToken: true,
20 defaultExpectedStatus: HttpStatusCode.OK_200
21 })
22 }
23
24 generate (options: OverrideCommandOptions = {}) {
25 const path = '/api/v1/runners/registration-tokens/generate'
26
27 return this.postBodyRequest({
28 ...options,
29
30 path,
31 implicitToken: true,
32 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
33 })
34 }
35
36 delete (options: OverrideCommandOptions & {
37 id: number
38 }) {
39 const path = '/api/v1/runners/registration-tokens/' + options.id
40
41 return this.deleteRequest({
42 ...options,
43
44 path,
45 implicitToken: true,
46 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
47 })
48 }
49
50 async getFirstRegistrationToken (options: OverrideCommandOptions = {}) {
51 const { data } = await this.list(options)
52
53 return data[0].registrationToken
54 }
55}
diff --git a/shared/server-commands/runners/runners-command.ts b/shared/server-commands/runners/runners-command.ts
new file mode 100644
index 000000000..ca9a1d7a3
--- /dev/null
+++ b/shared/server-commands/runners/runners-command.ts
@@ -0,0 +1,77 @@
1import { pick } from '@shared/core-utils'
2import { HttpStatusCode, RegisterRunnerBody, RegisterRunnerResult, ResultList, Runner, UnregisterRunnerBody } from '@shared/models'
3import { unwrapBody } from '../requests'
4import { AbstractCommand, OverrideCommandOptions } from '../shared'
5
6export class RunnersCommand extends AbstractCommand {
7
8 list (options: OverrideCommandOptions & {
9 start?: number
10 count?: number
11 sort?: string
12 } = {}) {
13 const path = '/api/v1/runners'
14
15 return this.getRequestBody<ResultList<Runner>>({
16 ...options,
17
18 path,
19 query: pick(options, [ 'start', 'count', 'sort' ]),
20 implicitToken: true,
21 defaultExpectedStatus: HttpStatusCode.OK_200
22 })
23 }
24
25 register (options: OverrideCommandOptions & RegisterRunnerBody) {
26 const path = '/api/v1/runners/register'
27
28 return unwrapBody<RegisterRunnerResult>(this.postBodyRequest({
29 ...options,
30
31 path,
32 fields: pick(options, [ 'name', 'registrationToken', 'description' ]),
33 implicitToken: true,
34 defaultExpectedStatus: HttpStatusCode.OK_200
35 }))
36 }
37
38 unregister (options: OverrideCommandOptions & UnregisterRunnerBody) {
39 const path = '/api/v1/runners/unregister'
40
41 return this.postBodyRequest({
42 ...options,
43
44 path,
45 fields: pick(options, [ 'runnerToken' ]),
46 implicitToken: false,
47 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
48 })
49 }
50
51 delete (options: OverrideCommandOptions & {
52 id: number
53 }) {
54 const path = '/api/v1/runners/' + options.id
55
56 return this.deleteRequest({
57 ...options,
58
59 path,
60 implicitToken: true,
61 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
62 })
63 }
64
65 // ---------------------------------------------------------------------------
66
67 async autoRegisterRunner () {
68 const { data } = await this.server.runnerRegistrationTokens.list({ sort: 'createdAt' })
69
70 const { runnerToken } = await this.register({
71 name: 'runner',
72 registrationToken: data[0].registrationToken
73 })
74
75 return runnerToken
76 }
77}
diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts
index 303fcab88..9a6e413f2 100644
--- a/shared/server-commands/server/config-command.ts
+++ b/shared/server-commands/server/config-command.ts
@@ -5,8 +5,9 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared/abstract-comm
5 5
6export class ConfigCommand extends AbstractCommand { 6export class ConfigCommand extends AbstractCommand {
7 7
8 static getCustomConfigResolutions (enabled: boolean) { 8 static getCustomConfigResolutions (enabled: boolean, with0p = false) {
9 return { 9 return {
10 '0p': enabled && with0p,
10 '144p': enabled, 11 '144p': enabled,
11 '240p': enabled, 12 '240p': enabled,
12 '360p': enabled, 13 '360p': enabled,
@@ -129,7 +130,8 @@ export class ConfigCommand extends AbstractCommand {
129 }) 130 })
130 } 131 }
131 132
132 enableTranscoding (webtorrent = true, hls = true) { 133 // TODO: convert args to object
134 enableTranscoding (webtorrent = true, hls = true, with0p = false) {
133 return this.updateExistingSubConfig({ 135 return this.updateExistingSubConfig({
134 newConfig: { 136 newConfig: {
135 transcoding: { 137 transcoding: {
@@ -138,7 +140,7 @@ export class ConfigCommand extends AbstractCommand {
138 allowAudioFiles: true, 140 allowAudioFiles: true,
139 allowAdditionalExtensions: true, 141 allowAdditionalExtensions: true,
140 142
141 resolutions: ConfigCommand.getCustomConfigResolutions(true), 143 resolutions: ConfigCommand.getCustomConfigResolutions(true, with0p),
142 144
143 webtorrent: { 145 webtorrent: {
144 enabled: webtorrent 146 enabled: webtorrent
@@ -151,6 +153,7 @@ export class ConfigCommand extends AbstractCommand {
151 }) 153 })
152 } 154 }
153 155
156 // TODO: convert args to object
154 enableMinimumTranscoding (webtorrent = true, hls = true) { 157 enableMinimumTranscoding (webtorrent = true, hls = true) {
155 return this.updateExistingSubConfig({ 158 return this.updateExistingSubConfig({
156 newConfig: { 159 newConfig: {
@@ -173,6 +176,25 @@ export class ConfigCommand extends AbstractCommand {
173 }) 176 })
174 } 177 }
175 178
179 enableRemoteTranscoding () {
180 return this.updateExistingSubConfig({
181 newConfig: {
182 transcoding: {
183 remoteRunners: {
184 enabled: true
185 }
186 },
187 live: {
188 transcoding: {
189 remoteRunners: {
190 enabled: true
191 }
192 }
193 }
194 }
195 })
196 }
197
176 // --------------------------------------------------------------------------- 198 // ---------------------------------------------------------------------------
177 199
178 enableStudio () { 200 enableStudio () {
@@ -363,6 +385,9 @@ export class ConfigCommand extends AbstractCommand {
363 }, 385 },
364 transcoding: { 386 transcoding: {
365 enabled: true, 387 enabled: true,
388 remoteRunners: {
389 enabled: false
390 },
366 allowAdditionalExtensions: true, 391 allowAdditionalExtensions: true,
367 allowAudioFiles: true, 392 allowAudioFiles: true,
368 threads: 1, 393 threads: 1,
@@ -398,6 +423,9 @@ export class ConfigCommand extends AbstractCommand {
398 maxUserLives: 50, 423 maxUserLives: 50,
399 transcoding: { 424 transcoding: {
400 enabled: true, 425 enabled: true,
426 remoteRunners: {
427 enabled: false
428 },
401 threads: 4, 429 threads: 4,
402 profile: 'default', 430 profile: 'default',
403 resolutions: { 431 resolutions: {
diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts
index e1d6cdff4..ff3098063 100644
--- a/shared/server-commands/server/jobs.ts
+++ b/shared/server-commands/server/jobs.ts
@@ -1,16 +1,17 @@
1 1
2import { expect } from 'chai' 2import { expect } from 'chai'
3import { wait } from '@shared/core-utils' 3import { wait } from '@shared/core-utils'
4import { JobState, JobType } from '../../models' 4import { JobState, JobType, RunnerJobState } from '../../models'
5import { PeerTubeServer } from './server' 5import { PeerTubeServer } from './server'
6 6
7async function waitJobs ( 7async function waitJobs (
8 serversArg: PeerTubeServer[] | PeerTubeServer, 8 serversArg: PeerTubeServer[] | PeerTubeServer,
9 options: { 9 options: {
10 skipDelayed?: boolean // default false 10 skipDelayed?: boolean // default false
11 runnerJobs?: boolean // default false
11 } = {} 12 } = {}
12) { 13) {
13 const { skipDelayed = false } = options 14 const { skipDelayed = false, runnerJobs = false } = options
14 15
15 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT 16 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT
16 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) 17 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10)
@@ -33,7 +34,8 @@ async function waitJobs (
33 // Check if each server has pending request 34 // Check if each server has pending request
34 for (const server of servers) { 35 for (const server of servers) {
35 for (const state of states) { 36 for (const state of states) {
36 const p = server.jobs.list({ 37
38 const jobPromise = server.jobs.list({
37 state, 39 state,
38 start: 0, 40 start: 0,
39 count: 10, 41 count: 10,
@@ -46,17 +48,29 @@ async function waitJobs (
46 } 48 }
47 }) 49 })
48 50
49 tasks.push(p) 51 tasks.push(jobPromise)
50 } 52 }
51 53
52 const p = server.debug.getDebug() 54 const debugPromise = server.debug.getDebug()
53 .then(obj => { 55 .then(obj => {
54 if (obj.activityPubMessagesWaiting !== 0) { 56 if (obj.activityPubMessagesWaiting !== 0) {
55 pendingRequests = true 57 pendingRequests = true
56 } 58 }
57 }) 59 })
60 tasks.push(debugPromise)
61
62 if (runnerJobs) {
63 const runnerJobsPromise = server.runnerJobs.list({ count: 100 })
64 .then(({ data }) => {
65 for (const job of data) {
66 if (job.state.id !== RunnerJobState.COMPLETED) {
67 pendingRequests = true
68 }
69 }
70 })
71 tasks.push(runnerJobsPromise)
72 }
58 73
59 tasks.push(p)
60 } 74 }
61 75
62 return tasks 76 return tasks
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index d7e751581..f68b81367 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -8,9 +8,9 @@ import { CLICommand } from '../cli'
8import { CustomPagesCommand } from '../custom-pages' 8import { CustomPagesCommand } from '../custom-pages'
9import { FeedCommand } from '../feeds' 9import { FeedCommand } from '../feeds'
10import { LogsCommand } from '../logs' 10import { LogsCommand } from '../logs'
11import { SQLCommand } from '../miscs'
12import { AbusesCommand } from '../moderation' 11import { AbusesCommand } from '../moderation'
13import { OverviewsCommand } from '../overviews' 12import { OverviewsCommand } from '../overviews'
13import { RunnerJobsCommand, RunnerRegistrationTokensCommand, RunnersCommand } from '../runners'
14import { SearchCommand } from '../search' 14import { SearchCommand } from '../search'
15import { SocketIOCommand } from '../socket' 15import { SocketIOCommand } from '../socket'
16import { 16import {
@@ -136,7 +136,6 @@ export class PeerTubeServer {
136 streamingPlaylists?: StreamingPlaylistsCommand 136 streamingPlaylists?: StreamingPlaylistsCommand
137 channels?: ChannelsCommand 137 channels?: ChannelsCommand
138 comments?: CommentsCommand 138 comments?: CommentsCommand
139 sql?: SQLCommand
140 notifications?: NotificationsCommand 139 notifications?: NotificationsCommand
141 servers?: ServersCommand 140 servers?: ServersCommand
142 login?: LoginCommand 141 login?: LoginCommand
@@ -150,6 +149,10 @@ export class PeerTubeServer {
150 videoToken?: VideoTokenCommand 149 videoToken?: VideoTokenCommand
151 registrations?: RegistrationsCommand 150 registrations?: RegistrationsCommand
152 151
152 runners?: RunnersCommand
153 runnerRegistrationTokens?: RunnerRegistrationTokensCommand
154 runnerJobs?: RunnerJobsCommand
155
153 constructor (options: { serverNumber: number } | { url: string }) { 156 constructor (options: { serverNumber: number } | { url: string }) {
154 if ((options as any).url) { 157 if ((options as any).url) {
155 this.setUrl((options as any).url) 158 this.setUrl((options as any).url)
@@ -311,14 +314,14 @@ export class PeerTubeServer {
311 }) 314 })
312 } 315 }
313 316
314 async kill () { 317 kill () {
315 if (!this.app) return 318 if (!this.app) return Promise.resolve()
316
317 await this.sql.cleanup()
318 319
319 process.kill(-this.app.pid) 320 process.kill(-this.app.pid)
320 321
321 this.app = null 322 this.app = null
323
324 return Promise.resolve()
322 } 325 }
323 326
324 private randomServer () { 327 private randomServer () {
@@ -420,7 +423,6 @@ export class PeerTubeServer {
420 this.streamingPlaylists = new StreamingPlaylistsCommand(this) 423 this.streamingPlaylists = new StreamingPlaylistsCommand(this)
421 this.channels = new ChannelsCommand(this) 424 this.channels = new ChannelsCommand(this)
422 this.comments = new CommentsCommand(this) 425 this.comments = new CommentsCommand(this)
423 this.sql = new SQLCommand(this)
424 this.notifications = new NotificationsCommand(this) 426 this.notifications = new NotificationsCommand(this)
425 this.servers = new ServersCommand(this) 427 this.servers = new ServersCommand(this)
426 this.login = new LoginCommand(this) 428 this.login = new LoginCommand(this)
@@ -433,5 +435,9 @@ export class PeerTubeServer {
433 this.twoFactor = new TwoFactorCommand(this) 435 this.twoFactor = new TwoFactorCommand(this)
434 this.videoToken = new VideoTokenCommand(this) 436 this.videoToken = new VideoTokenCommand(this)
435 this.registrations = new RegistrationsCommand(this) 437 this.registrations = new RegistrationsCommand(this)
438
439 this.runners = new RunnersCommand(this)
440 this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
441 this.runnerJobs = new RunnerJobsCommand(this)
436 } 442 }
437} 443}
diff --git a/shared/server-commands/server/servers.ts b/shared/server-commands/server/servers.ts
index b2b61adb3..fe9da9e63 100644
--- a/shared/server-commands/server/servers.ts
+++ b/shared/server-commands/server/servers.ts
@@ -20,7 +20,7 @@ function createMultipleServers (totalServers: number, configOverride?: object, o
20 return Promise.all(serverPromises) 20 return Promise.all(serverPromises)
21} 21}
22 22
23async function killallServers (servers: PeerTubeServer[]) { 23function killallServers (servers: PeerTubeServer[]) {
24 return Promise.all(servers.map(s => s.kill())) 24 return Promise.all(servers.map(s => s.kill()))
25} 25}
26 26
diff --git a/shared/server-commands/shared/abstract-command.ts b/shared/server-commands/shared/abstract-command.ts
index 1b53a5330..ca4ffada9 100644
--- a/shared/server-commands/shared/abstract-command.ts
+++ b/shared/server-commands/shared/abstract-command.ts
@@ -33,6 +33,7 @@ interface InternalCommonCommandOptions extends OverrideCommandOptions {
33 host?: string 33 host?: string
34 headers?: { [ name: string ]: string } 34 headers?: { [ name: string ]: string }
35 requestType?: string 35 requestType?: string
36 responseType?: string
36 xForwardedFor?: string 37 xForwardedFor?: string
37} 38}
38 39
@@ -169,7 +170,7 @@ abstract class AbstractCommand {
169 } 170 }
170 171
171 protected buildCommonRequestOptions (options: InternalCommonCommandOptions) { 172 protected buildCommonRequestOptions (options: InternalCommonCommandOptions) {
172 const { url, path, redirects, contentType, accept, range, host, headers, requestType, xForwardedFor } = options 173 const { url, path, redirects, contentType, accept, range, host, headers, requestType, xForwardedFor, responseType } = options
173 174
174 return { 175 return {
175 url: url ?? this.server.url, 176 url: url ?? this.server.url,
@@ -185,6 +186,7 @@ abstract class AbstractCommand {
185 accept, 186 accept,
186 headers, 187 headers,
187 type: requestType, 188 type: requestType,
189 responseType,
188 xForwardedFor 190 xForwardedFor
189 } 191 }
190 } 192 }
diff --git a/shared/server-commands/socket/socket-io-command.ts b/shared/server-commands/socket/socket-io-command.ts
index c277ead28..c28a86366 100644
--- a/shared/server-commands/socket/socket-io-command.ts
+++ b/shared/server-commands/socket/socket-io-command.ts
@@ -12,4 +12,13 @@ export class SocketIOCommand extends AbstractCommand {
12 getLiveNotificationSocket () { 12 getLiveNotificationSocket () {
13 return io(this.server.url + '/live-videos') 13 return io(this.server.url + '/live-videos')
14 } 14 }
15
16 getRunnersSocket (options: {
17 runnerToken: string
18 }) {
19 return io(this.server.url + '/runners', {
20 reconnection: false,
21 auth: { runnerToken: options.runnerToken }
22 })
23 }
15} 24}
diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts
index 3273e3a8f..dc3c5a86e 100644
--- a/shared/server-commands/videos/live-command.ts
+++ b/shared/server-commands/videos/live-command.ts
@@ -121,7 +121,7 @@ export class LiveCommand extends AbstractCommand {
121 permanentLive: boolean 121 permanentLive: boolean
122 privacy?: VideoPrivacy 122 privacy?: VideoPrivacy
123 }) { 123 }) {
124 const { saveReplay, permanentLive, privacy } = options 124 const { saveReplay, permanentLive, privacy = VideoPrivacy.PUBLIC } = options
125 125
126 const { uuid } = await this.create({ 126 const { uuid } = await this.create({
127 ...options, 127 ...options,
diff --git a/shared/server-commands/videos/streaming-playlists-command.ts b/shared/server-commands/videos/streaming-playlists-command.ts
index 26ab2735f..7b92dcc0a 100644
--- a/shared/server-commands/videos/streaming-playlists-command.ts
+++ b/shared/server-commands/videos/streaming-playlists-command.ts
@@ -13,7 +13,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
13 13
14 withRetry?: boolean // default false 14 withRetry?: boolean // default false
15 currentRetry?: number 15 currentRetry?: number
16 }) { 16 }): Promise<string> {
17 const { videoFileToken, reinjectVideoFileToken, withRetry, currentRetry = 1 } = options 17 const { videoFileToken, reinjectVideoFileToken, withRetry, currentRetry = 1 } = options
18 18
19 try { 19 try {
@@ -54,6 +54,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
54 url: options.url, 54 url: options.url,
55 range: options.range, 55 range: options.range,
56 implicitToken: false, 56 implicitToken: false,
57 responseType: 'application/octet-stream',
57 defaultExpectedStatus: HttpStatusCode.OK_200 58 defaultExpectedStatus: HttpStatusCode.OK_200
58 })) 59 }))
59 } 60 }
@@ -65,6 +66,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
65 ...options, 66 ...options,
66 67
67 url: options.url, 68 url: options.url,
69 contentType: 'application/json',
68 implicitToken: false, 70 implicitToken: false,
69 defaultExpectedStatus: HttpStatusCode.OK_200 71 defaultExpectedStatus: HttpStatusCode.OK_200
70 })) 72 }))