diff options
Diffstat (limited to 'shared')
21 files changed, 299 insertions, 17 deletions
diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index 7fc92f804..aa2c8e8fa 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts | |||
@@ -16,6 +16,10 @@ function dateIsValid (dateString: string, interval = 300000) { | |||
16 | return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval | 16 | return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval |
17 | } | 17 | } |
18 | 18 | ||
19 | function expectStartWith (str: string, start: string) { | ||
20 | expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true | ||
21 | } | ||
22 | |||
19 | async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { | 23 | async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { |
20 | const res = await makeGetRequest({ | 24 | const res = await makeGetRequest({ |
21 | url, | 25 | url, |
@@ -42,5 +46,6 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f | |||
42 | export { | 46 | export { |
43 | dateIsValid, | 47 | dateIsValid, |
44 | testImage, | 48 | testImage, |
45 | testFileExistsOrNot | 49 | testFileExistsOrNot, |
50 | expectStartWith | ||
46 | } | 51 | } |
diff --git a/shared/extra-utils/miscs/tests.ts b/shared/extra-utils/miscs/tests.ts index 3dfb2487e..dd86041fe 100644 --- a/shared/extra-utils/miscs/tests.ts +++ b/shared/extra-utils/miscs/tests.ts | |||
@@ -28,7 +28,9 @@ const FIXTURE_URLS = { | |||
28 | 28 | ||
29 | badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', | 29 | badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', |
30 | goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', | 30 | goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', |
31 | video4K: 'https://download.cpy.re/peertube/4k_file.txt' | 31 | goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4', |
32 | |||
33 | file4K: 'https://download.cpy.re/peertube/4k_file.txt' | ||
32 | } | 34 | } |
33 | 35 | ||
34 | function parallelTests () { | 36 | function parallelTests () { |
@@ -42,7 +44,15 @@ function isGithubCI () { | |||
42 | function areHttpImportTestsDisabled () { | 44 | function areHttpImportTestsDisabled () { |
43 | const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' | 45 | const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' |
44 | 46 | ||
45 | if (disabled) console.log('Import tests are disabled') | 47 | if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled') |
48 | |||
49 | return disabled | ||
50 | } | ||
51 | |||
52 | function areObjectStorageTestsDisabled () { | ||
53 | const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' | ||
54 | |||
55 | if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') | ||
46 | 56 | ||
47 | return disabled | 57 | return disabled |
48 | } | 58 | } |
@@ -89,6 +99,7 @@ export { | |||
89 | buildAbsoluteFixturePath, | 99 | buildAbsoluteFixturePath, |
90 | getFileSize, | 100 | getFileSize, |
91 | buildRequestStub, | 101 | buildRequestStub, |
102 | areObjectStorageTestsDisabled, | ||
92 | wait, | 103 | wait, |
93 | root | 104 | root |
94 | } | 105 | } |
diff --git a/shared/extra-utils/mock-servers/index.ts b/shared/extra-utils/mock-servers/index.ts index 0ec07f685..93c00c788 100644 --- a/shared/extra-utils/mock-servers/index.ts +++ b/shared/extra-utils/mock-servers/index.ts | |||
@@ -2,3 +2,4 @@ export * from './mock-email' | |||
2 | export * from './mock-instances-index' | 2 | export * from './mock-instances-index' |
3 | export * from './mock-joinpeertube-versions' | 3 | export * from './mock-joinpeertube-versions' |
4 | export * from './mock-plugin-blocklist' | 4 | export * from './mock-plugin-blocklist' |
5 | export * from './mock-object-storage' | ||
diff --git a/shared/extra-utils/mock-servers/mock-object-storage.ts b/shared/extra-utils/mock-servers/mock-object-storage.ts new file mode 100644 index 000000000..19ea7c87c --- /dev/null +++ b/shared/extra-utils/mock-servers/mock-object-storage.ts | |||
@@ -0,0 +1,42 @@ | |||
1 | import * as express from 'express' | ||
2 | import got, { RequestError } from 'got' | ||
3 | import { Server } from 'http' | ||
4 | import { pipeline } from 'stream' | ||
5 | import { randomInt } from '@shared/core-utils' | ||
6 | import { ObjectStorageCommand } from '../server' | ||
7 | |||
8 | export class MockObjectStorage { | ||
9 | private server: Server | ||
10 | |||
11 | initialize () { | ||
12 | return new Promise<number>(res => { | ||
13 | const app = express() | ||
14 | |||
15 | app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
16 | const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` | ||
17 | |||
18 | if (process.env.DEBUG) { | ||
19 | console.log('Receiving request on mocked server %s.', req.url) | ||
20 | console.log('Proxifying request to %s', url) | ||
21 | } | ||
22 | |||
23 | return pipeline( | ||
24 | got.stream(url, { throwHttpErrors: false }), | ||
25 | res, | ||
26 | (err: RequestError) => { | ||
27 | if (!err) return | ||
28 | |||
29 | console.error('Pipeline failed.', err) | ||
30 | } | ||
31 | ) | ||
32 | }) | ||
33 | |||
34 | const port = 42301 + randomInt(1, 100) | ||
35 | this.server = app.listen(port, () => res(port)) | ||
36 | }) | ||
37 | } | ||
38 | |||
39 | terminate () { | ||
40 | if (this.server) this.server.close() | ||
41 | } | ||
42 | } | ||
diff --git a/shared/extra-utils/requests/requests.ts b/shared/extra-utils/requests/requests.ts index 70f790222..e3ecd1af2 100644 --- a/shared/extra-utils/requests/requests.ts +++ b/shared/extra-utils/requests/requests.ts | |||
@@ -121,6 +121,20 @@ function unwrapText (test: request.Test): Promise<string> { | |||
121 | return test.then(res => res.text) | 121 | return test.then(res => res.text) |
122 | } | 122 | } |
123 | 123 | ||
124 | function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> { | ||
125 | return test.then(res => { | ||
126 | if (res.body instanceof Buffer) { | ||
127 | return JSON.parse(new TextDecoder().decode(res.body)) | ||
128 | } | ||
129 | |||
130 | return res.body | ||
131 | }) | ||
132 | } | ||
133 | |||
134 | function unwrapTextOrDecode (test: request.Test): Promise<string> { | ||
135 | return test.then(res => res.text || new TextDecoder().decode(res.body)) | ||
136 | } | ||
137 | |||
124 | // --------------------------------------------------------------------------- | 138 | // --------------------------------------------------------------------------- |
125 | 139 | ||
126 | export { | 140 | export { |
@@ -134,6 +148,8 @@ export { | |||
134 | makeRawRequest, | 148 | makeRawRequest, |
135 | makeActivityPubGetRequest, | 149 | makeActivityPubGetRequest, |
136 | unwrapBody, | 150 | unwrapBody, |
151 | unwrapTextOrDecode, | ||
152 | unwrapBodyOrDecodeToJSON, | ||
137 | unwrapText | 153 | unwrapText |
138 | } | 154 | } |
139 | 155 | ||
diff --git a/shared/extra-utils/server/config-command.ts b/shared/extra-utils/server/config-command.ts index 11148aa46..51d04fa63 100644 --- a/shared/extra-utils/server/config-command.ts +++ b/shared/extra-utils/server/config-command.ts | |||
@@ -18,6 +18,70 @@ export class ConfigCommand extends AbstractCommand { | |||
18 | } | 18 | } |
19 | } | 19 | } |
20 | 20 | ||
21 | enableImports () { | ||
22 | return this.updateExistingSubConfig({ | ||
23 | newConfig: { | ||
24 | import: { | ||
25 | videos: { | ||
26 | http: { | ||
27 | enabled: true | ||
28 | }, | ||
29 | |||
30 | torrent: { | ||
31 | enabled: true | ||
32 | } | ||
33 | } | ||
34 | } | ||
35 | } | ||
36 | }) | ||
37 | } | ||
38 | |||
39 | enableLive (options: { | ||
40 | allowReplay?: boolean | ||
41 | transcoding?: boolean | ||
42 | } = {}) { | ||
43 | return this.updateExistingSubConfig({ | ||
44 | newConfig: { | ||
45 | live: { | ||
46 | enabled: true, | ||
47 | allowReplay: options.allowReplay ?? true, | ||
48 | transcoding: { | ||
49 | enabled: options.transcoding ?? true, | ||
50 | resolutions: ConfigCommand.getCustomConfigResolutions(true) | ||
51 | } | ||
52 | } | ||
53 | } | ||
54 | }) | ||
55 | } | ||
56 | |||
57 | disableTranscoding () { | ||
58 | return this.updateExistingSubConfig({ | ||
59 | newConfig: { | ||
60 | transcoding: { | ||
61 | enabled: false | ||
62 | } | ||
63 | } | ||
64 | }) | ||
65 | } | ||
66 | |||
67 | enableTranscoding (webtorrent = true, hls = true) { | ||
68 | return this.updateExistingSubConfig({ | ||
69 | newConfig: { | ||
70 | transcoding: { | ||
71 | enabled: true, | ||
72 | resolutions: ConfigCommand.getCustomConfigResolutions(true), | ||
73 | |||
74 | webtorrent: { | ||
75 | enabled: webtorrent | ||
76 | }, | ||
77 | hls: { | ||
78 | enabled: hls | ||
79 | } | ||
80 | } | ||
81 | } | ||
82 | }) | ||
83 | } | ||
84 | |||
21 | getConfig (options: OverrideCommandOptions = {}) { | 85 | getConfig (options: OverrideCommandOptions = {}) { |
22 | const path = '/api/v1/config' | 86 | const path = '/api/v1/config' |
23 | 87 | ||
@@ -81,6 +145,14 @@ export class ConfigCommand extends AbstractCommand { | |||
81 | }) | 145 | }) |
82 | } | 146 | } |
83 | 147 | ||
148 | async updateExistingSubConfig (options: OverrideCommandOptions & { | ||
149 | newConfig: DeepPartial<CustomConfig> | ||
150 | }) { | ||
151 | const existing = await this.getCustomConfig(options) | ||
152 | |||
153 | return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) | ||
154 | } | ||
155 | |||
84 | updateCustomSubConfig (options: OverrideCommandOptions & { | 156 | updateCustomSubConfig (options: OverrideCommandOptions & { |
85 | newConfig: DeepPartial<CustomConfig> | 157 | newConfig: DeepPartial<CustomConfig> |
86 | }) { | 158 | }) { |
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts index 9055dfc57..92ff7a0f9 100644 --- a/shared/extra-utils/server/index.ts +++ b/shared/extra-utils/server/index.ts | |||
@@ -6,6 +6,7 @@ export * from './follows-command' | |||
6 | export * from './follows' | 6 | export * from './follows' |
7 | export * from './jobs' | 7 | export * from './jobs' |
8 | export * from './jobs-command' | 8 | export * from './jobs-command' |
9 | export * from './object-storage-command' | ||
9 | export * from './plugins-command' | 10 | export * from './plugins-command' |
10 | export * from './plugins' | 11 | export * from './plugins' |
11 | export * from './redundancy-command' | 12 | export * from './redundancy-command' |
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts index c4eb12dc2..91771c176 100644 --- a/shared/extra-utils/server/jobs-command.ts +++ b/shared/extra-utils/server/jobs-command.ts | |||
@@ -5,6 +5,16 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared' | |||
5 | 5 | ||
6 | export class JobsCommand extends AbstractCommand { | 6 | export class JobsCommand extends AbstractCommand { |
7 | 7 | ||
8 | async getLatest (options: OverrideCommandOptions & { | ||
9 | jobType: JobType | ||
10 | }) { | ||
11 | const { data } = await this.getJobsList({ ...options, start: 0, count: 1, sort: '-createdAt' }) | ||
12 | |||
13 | if (data.length === 0) return undefined | ||
14 | |||
15 | return data[0] | ||
16 | } | ||
17 | |||
8 | getJobsList (options: OverrideCommandOptions & { | 18 | getJobsList (options: OverrideCommandOptions & { |
9 | state?: JobState | 19 | state?: JobState |
10 | jobType?: JobType | 20 | jobType?: JobType |
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index 64a0353eb..27104bfdf 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts | |||
@@ -3,7 +3,7 @@ import { JobState } from '../../models' | |||
3 | import { wait } from '../miscs' | 3 | import { wait } from '../miscs' |
4 | import { PeerTubeServer } from './server' | 4 | import { PeerTubeServer } from './server' |
5 | 5 | ||
6 | async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { | 6 | async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer, skipDelayed = false) { |
7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT | 7 | const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT |
8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) | 8 | ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) |
9 | : 250 | 9 | : 250 |
@@ -13,7 +13,9 @@ async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { | |||
13 | if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] | 13 | if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] |
14 | else servers = serversArg as PeerTubeServer[] | 14 | else servers = serversArg as PeerTubeServer[] |
15 | 15 | ||
16 | const states: JobState[] = [ 'waiting', 'active', 'delayed' ] | 16 | const states: JobState[] = [ 'waiting', 'active' ] |
17 | if (!skipDelayed) states.push('delayed') | ||
18 | |||
17 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] | 19 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] |
18 | let pendingRequests: boolean | 20 | let pendingRequests: boolean |
19 | 21 | ||
diff --git a/shared/extra-utils/server/object-storage-command.ts b/shared/extra-utils/server/object-storage-command.ts new file mode 100644 index 000000000..b4de8f4cb --- /dev/null +++ b/shared/extra-utils/server/object-storage-command.ts | |||
@@ -0,0 +1,77 @@ | |||
1 | |||
2 | import { HttpStatusCode } from '@shared/models' | ||
3 | import { makePostBodyRequest } from '../requests' | ||
4 | import { AbstractCommand } from '../shared' | ||
5 | |||
6 | export class ObjectStorageCommand extends AbstractCommand { | ||
7 | static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists' | ||
8 | static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos' | ||
9 | |||
10 | static getDefaultConfig () { | ||
11 | return { | ||
12 | object_storage: { | ||
13 | enabled: true, | ||
14 | endpoint: 'http://' + this.getEndpointHost(), | ||
15 | region: this.getRegion(), | ||
16 | |||
17 | credentials: this.getCredentialsConfig(), | ||
18 | |||
19 | streaming_playlists: { | ||
20 | bucket_name: this.DEFAULT_PLAYLIST_BUCKET | ||
21 | }, | ||
22 | |||
23 | videos: { | ||
24 | bucket_name: this.DEFAULT_WEBTORRENT_BUCKET | ||
25 | } | ||
26 | } | ||
27 | } | ||
28 | } | ||
29 | |||
30 | static getCredentialsConfig () { | ||
31 | return { | ||
32 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
33 | secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
34 | } | ||
35 | } | ||
36 | |||
37 | static getEndpointHost () { | ||
38 | return 'localhost:9444' | ||
39 | } | ||
40 | |||
41 | static getRegion () { | ||
42 | return 'us-east-1' | ||
43 | } | ||
44 | |||
45 | static getWebTorrentBaseUrl () { | ||
46 | return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/` | ||
47 | } | ||
48 | |||
49 | static getPlaylistBaseUrl () { | ||
50 | return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/` | ||
51 | } | ||
52 | |||
53 | static async prepareDefaultBuckets () { | ||
54 | await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET) | ||
55 | await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET) | ||
56 | } | ||
57 | |||
58 | static async createBucket (name: string) { | ||
59 | await makePostBodyRequest({ | ||
60 | url: this.getEndpointHost(), | ||
61 | path: '/ui/' + name + '?delete', | ||
62 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
63 | }) | ||
64 | |||
65 | await makePostBodyRequest({ | ||
66 | url: this.getEndpointHost(), | ||
67 | path: '/ui/' + name + '?create', | ||
68 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
69 | }) | ||
70 | |||
71 | await makePostBodyRequest({ | ||
72 | url: this.getEndpointHost(), | ||
73 | path: '/ui/' + name + '?make-public', | ||
74 | expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 | ||
75 | }) | ||
76 | } | ||
77 | } | ||
diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 3c335b8e4..bc5e1cd5f 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts | |||
@@ -38,11 +38,13 @@ import { PluginsCommand } from './plugins-command' | |||
38 | import { RedundancyCommand } from './redundancy-command' | 38 | import { RedundancyCommand } from './redundancy-command' |
39 | import { ServersCommand } from './servers-command' | 39 | import { ServersCommand } from './servers-command' |
40 | import { StatsCommand } from './stats-command' | 40 | import { StatsCommand } from './stats-command' |
41 | import { ObjectStorageCommand } from './object-storage-command' | ||
41 | 42 | ||
42 | export type RunServerOptions = { | 43 | export type RunServerOptions = { |
43 | hideLogs?: boolean | 44 | hideLogs?: boolean |
44 | nodeArgs?: string[] | 45 | nodeArgs?: string[] |
45 | peertubeArgs?: string[] | 46 | peertubeArgs?: string[] |
47 | env?: { [ id: string ]: string } | ||
46 | } | 48 | } |
47 | 49 | ||
48 | export class PeerTubeServer { | 50 | export class PeerTubeServer { |
@@ -121,6 +123,7 @@ export class PeerTubeServer { | |||
121 | servers?: ServersCommand | 123 | servers?: ServersCommand |
122 | login?: LoginCommand | 124 | login?: LoginCommand |
123 | users?: UsersCommand | 125 | users?: UsersCommand |
126 | objectStorage?: ObjectStorageCommand | ||
124 | videos?: VideosCommand | 127 | videos?: VideosCommand |
125 | 128 | ||
126 | constructor (options: { serverNumber: number } | { url: string }) { | 129 | constructor (options: { serverNumber: number } | { url: string }) { |
@@ -202,6 +205,10 @@ export class PeerTubeServer { | |||
202 | env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() | 205 | env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() |
203 | env['NODE_CONFIG'] = JSON.stringify(configOverride) | 206 | env['NODE_CONFIG'] = JSON.stringify(configOverride) |
204 | 207 | ||
208 | if (options.env) { | ||
209 | Object.assign(env, options.env) | ||
210 | } | ||
211 | |||
205 | const forkOptions = { | 212 | const forkOptions = { |
206 | silent: true, | 213 | silent: true, |
207 | env, | 214 | env, |
@@ -209,10 +216,17 @@ export class PeerTubeServer { | |||
209 | execArgv: options.nodeArgs || [] | 216 | execArgv: options.nodeArgs || [] |
210 | } | 217 | } |
211 | 218 | ||
212 | return new Promise<void>(res => { | 219 | return new Promise<void>((res, rej) => { |
213 | const self = this | 220 | const self = this |
214 | 221 | ||
215 | this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) | 222 | this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) |
223 | |||
224 | const onExit = function () { | ||
225 | return rej(new Error('Process exited')) | ||
226 | } | ||
227 | |||
228 | this.app.on('exit', onExit) | ||
229 | |||
216 | this.app.stdout.on('data', function onStdout (data) { | 230 | this.app.stdout.on('data', function onStdout (data) { |
217 | let dontContinue = false | 231 | let dontContinue = false |
218 | 232 | ||
@@ -241,6 +255,7 @@ export class PeerTubeServer { | |||
241 | console.log(data.toString()) | 255 | console.log(data.toString()) |
242 | } else { | 256 | } else { |
243 | self.app.stdout.removeListener('data', onStdout) | 257 | self.app.stdout.removeListener('data', onStdout) |
258 | self.app.removeListener('exit', onExit) | ||
244 | } | 259 | } |
245 | 260 | ||
246 | process.on('exit', () => { | 261 | process.on('exit', () => { |
@@ -365,5 +380,6 @@ export class PeerTubeServer { | |||
365 | this.login = new LoginCommand(this) | 380 | this.login = new LoginCommand(this) |
366 | this.users = new UsersCommand(this) | 381 | this.users = new UsersCommand(this) |
367 | this.videos = new VideosCommand(this) | 382 | this.videos = new VideosCommand(this) |
383 | this.objectStorage = new ObjectStorageCommand(this) | ||
368 | } | 384 | } |
369 | } | 385 | } |
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index f0622feb0..21ab9405b 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts | |||
@@ -10,11 +10,11 @@ async function createSingleServer (serverNumber: number, configOverride?: Object | |||
10 | return server | 10 | return server |
11 | } | 11 | } |
12 | 12 | ||
13 | function createMultipleServers (totalServers: number, configOverride?: Object) { | 13 | function createMultipleServers (totalServers: number, configOverride?: Object, options: RunServerOptions = {}) { |
14 | const serverPromises: Promise<PeerTubeServer>[] = [] | 14 | const serverPromises: Promise<PeerTubeServer>[] = [] |
15 | 15 | ||
16 | for (let i = 1; i <= totalServers; i++) { | 16 | for (let i = 1; i <= totalServers; i++) { |
17 | serverPromises.push(createSingleServer(i, configOverride)) | 17 | serverPromises.push(createSingleServer(i, configOverride, options)) |
18 | } | 18 | } |
19 | 19 | ||
20 | return Promise.all(serverPromises) | 20 | return Promise.all(serverPromises) |
diff --git a/shared/extra-utils/videos/live-command.ts b/shared/extra-utils/videos/live-command.ts index 81ae458e0..74f5d3089 100644 --- a/shared/extra-utils/videos/live-command.ts +++ b/shared/extra-utils/videos/live-command.ts | |||
@@ -126,7 +126,7 @@ export class LiveCommand extends AbstractCommand { | |||
126 | video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) | 126 | video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) |
127 | 127 | ||
128 | await wait(500) | 128 | await wait(500) |
129 | } while (video.isLive === true && video.state.id !== VideoState.PUBLISHED) | 129 | } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED) |
130 | } | 130 | } |
131 | 131 | ||
132 | async countPlaylists (options: OverrideCommandOptions & { | 132 | async countPlaylists (options: OverrideCommandOptions & { |
diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts index 9a6df07a8..29f99ed6d 100644 --- a/shared/extra-utils/videos/live.ts +++ b/shared/extra-utils/videos/live.ts | |||
@@ -89,6 +89,12 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi | |||
89 | } | 89 | } |
90 | } | 90 | } |
91 | 91 | ||
92 | async function waitUntilLiveSavedOnAllServers (servers: PeerTubeServer[], videoId: string) { | ||
93 | for (const server of servers) { | ||
94 | await server.live.waitUntilSaved({ videoId }) | ||
95 | } | ||
96 | } | ||
97 | |||
92 | async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { | 98 | async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { |
93 | const basePath = server.servers.buildDirectory('streaming-playlists') | 99 | const basePath = server.servers.buildDirectory('streaming-playlists') |
94 | const hlsPath = join(basePath, 'hls', videoUUID) | 100 | const hlsPath = join(basePath, 'hls', videoUUID) |
@@ -126,5 +132,6 @@ export { | |||
126 | testFfmpegStreamError, | 132 | testFfmpegStreamError, |
127 | stopFfmpeg, | 133 | stopFfmpeg, |
128 | waitUntilLivePublishedOnAllServers, | 134 | waitUntilLivePublishedOnAllServers, |
135 | waitUntilLiveSavedOnAllServers, | ||
129 | checkLiveCleanupAfterSave | 136 | checkLiveCleanupAfterSave |
130 | } | 137 | } |
diff --git a/shared/extra-utils/videos/streaming-playlists-command.ts b/shared/extra-utils/videos/streaming-playlists-command.ts index 9662685da..5d40d35cb 100644 --- a/shared/extra-utils/videos/streaming-playlists-command.ts +++ b/shared/extra-utils/videos/streaming-playlists-command.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { HttpStatusCode } from '@shared/models' | 1 | import { HttpStatusCode } from '@shared/models' |
2 | import { unwrapBody, unwrapText } from '../requests' | 2 | import { unwrapBody, unwrapTextOrDecode, unwrapBodyOrDecodeToJSON } from '../requests' |
3 | import { AbstractCommand, OverrideCommandOptions } from '../shared' | 3 | import { AbstractCommand, OverrideCommandOptions } from '../shared' |
4 | 4 | ||
5 | export class StreamingPlaylistsCommand extends AbstractCommand { | 5 | export class StreamingPlaylistsCommand extends AbstractCommand { |
@@ -7,7 +7,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand { | |||
7 | get (options: OverrideCommandOptions & { | 7 | get (options: OverrideCommandOptions & { |
8 | url: string | 8 | url: string |
9 | }) { | 9 | }) { |
10 | return unwrapText(this.getRawRequest({ | 10 | return unwrapTextOrDecode(this.getRawRequest({ |
11 | ...options, | 11 | ...options, |
12 | 12 | ||
13 | url: options.url, | 13 | url: options.url, |
@@ -33,7 +33,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand { | |||
33 | getSegmentSha256 (options: OverrideCommandOptions & { | 33 | getSegmentSha256 (options: OverrideCommandOptions & { |
34 | url: string | 34 | url: string |
35 | }) { | 35 | }) { |
36 | return unwrapBody<{ [ id: string ]: string }>(this.getRawRequest({ | 36 | return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ |
37 | ...options, | 37 | ...options, |
38 | 38 | ||
39 | url: options.url, | 39 | url: options.url, |
diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index a224b8f5f..6671e3fa6 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts | |||
@@ -9,17 +9,16 @@ async function checkSegmentHash (options: { | |||
9 | server: PeerTubeServer | 9 | server: PeerTubeServer |
10 | baseUrlPlaylist: string | 10 | baseUrlPlaylist: string |
11 | baseUrlSegment: string | 11 | baseUrlSegment: string |
12 | videoUUID: string | ||
13 | resolution: number | 12 | resolution: number |
14 | hlsPlaylist: VideoStreamingPlaylist | 13 | hlsPlaylist: VideoStreamingPlaylist |
15 | }) { | 14 | }) { |
16 | const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options | 15 | const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist } = options |
17 | const command = server.streamingPlaylists | 16 | const command = server.streamingPlaylists |
18 | 17 | ||
19 | const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) | 18 | const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) |
20 | const videoName = basename(file.fileUrl) | 19 | const videoName = basename(file.fileUrl) |
21 | 20 | ||
22 | const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) | 21 | const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) |
23 | 22 | ||
24 | const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) | 23 | const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) |
25 | 24 | ||
@@ -28,7 +27,7 @@ async function checkSegmentHash (options: { | |||
28 | const range = `${offset}-${offset + length - 1}` | 27 | const range = `${offset}-${offset + length - 1}` |
29 | 28 | ||
30 | const segmentBody = await command.getSegment({ | 29 | const segmentBody = await command.getSegment({ |
31 | url: `${baseUrlSegment}/${videoUUID}/${videoName}`, | 30 | url: `${baseUrlSegment}/${videoName}`, |
32 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, | 31 | expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, |
33 | range: `bytes=${range}` | 32 | range: `bytes=${range}` |
34 | }) | 33 | }) |
diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 33725bfdc..d35339c8d 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts | |||
@@ -188,6 +188,17 @@ export class VideosCommand extends AbstractCommand { | |||
188 | return id | 188 | return id |
189 | } | 189 | } |
190 | 190 | ||
191 | async listFiles (options: OverrideCommandOptions & { | ||
192 | id: number | string | ||
193 | }) { | ||
194 | const video = await this.get(options) | ||
195 | |||
196 | const files = video.files || [] | ||
197 | const hlsFiles = video.streamingPlaylists[0]?.files || [] | ||
198 | |||
199 | return files.concat(hlsFiles) | ||
200 | } | ||
201 | |||
191 | // --------------------------------------------------------------------------- | 202 | // --------------------------------------------------------------------------- |
192 | 203 | ||
193 | listMyVideos (options: OverrideCommandOptions & { | 204 | listMyVideos (options: OverrideCommandOptions & { |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 4ab249e0b..ff96283a4 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -19,6 +19,7 @@ export type JobType = | |||
19 | | 'video-redundancy' | 19 | | 'video-redundancy' |
20 | | 'video-live-ending' | 20 | | 'video-live-ending' |
21 | | 'actor-keys' | 21 | | 'actor-keys' |
22 | | 'move-to-object-storage' | ||
22 | 23 | ||
23 | export interface Job { | 24 | export interface Job { |
24 | id: number | 25 | id: number |
@@ -136,3 +137,8 @@ export interface VideoLiveEndingPayload { | |||
136 | export interface ActorKeysPayload { | 137 | export interface ActorKeysPayload { |
137 | actorId: number | 138 | actorId: number |
138 | } | 139 | } |
140 | |||
141 | export interface MoveObjectStoragePayload { | ||
142 | videoUUID: string | ||
143 | isNewVideo: boolean | ||
144 | } | ||
diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index faa9b9868..733c433a0 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts | |||
@@ -26,6 +26,7 @@ export * from './video-resolution.enum' | |||
26 | export * from './video-schedule-update.model' | 26 | export * from './video-schedule-update.model' |
27 | export * from './video-sort-field.type' | 27 | export * from './video-sort-field.type' |
28 | export * from './video-state.enum' | 28 | export * from './video-state.enum' |
29 | export * from './video-storage.enum' | ||
29 | 30 | ||
30 | export * from './video-streaming-playlist.model' | 31 | export * from './video-streaming-playlist.model' |
31 | export * from './video-streaming-playlist.type' | 32 | export * from './video-streaming-playlist.type' |
diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts index 49d997f24..c6af481e7 100644 --- a/shared/models/videos/video-state.enum.ts +++ b/shared/models/videos/video-state.enum.ts | |||
@@ -3,5 +3,6 @@ export const enum VideoState { | |||
3 | TO_TRANSCODE = 2, | 3 | TO_TRANSCODE = 2, |
4 | TO_IMPORT = 3, | 4 | TO_IMPORT = 3, |
5 | WAITING_FOR_LIVE = 4, | 5 | WAITING_FOR_LIVE = 4, |
6 | LIVE_ENDED = 5 | 6 | LIVE_ENDED = 5, |
7 | TO_MOVE_TO_EXTERNAL_STORAGE = 6 | ||
7 | } | 8 | } |
diff --git a/shared/models/videos/video-storage.enum.ts b/shared/models/videos/video-storage.enum.ts new file mode 100644 index 000000000..7c6690db2 --- /dev/null +++ b/shared/models/videos/video-storage.enum.ts | |||
@@ -0,0 +1,4 @@ | |||
1 | export const enum VideoStorage { | ||
2 | FILE_SYSTEM, | ||
3 | OBJECT_STORAGE, | ||
4 | } | ||