aboutsummaryrefslogtreecommitdiffhomepage
path: root/shared/extra-utils
diff options
context:
space:
mode:
authorJelle Besseling <jelle@pingiun.com>2021-08-17 08:26:20 +0200
committerGitHub <noreply@github.com>2021-08-17 08:26:20 +0200
commit0305db28c98fd6cf43a3c50ba92c76215e99d512 (patch)
tree33b753a19728d9f453c1aa4f19b36ac797e5fe80 /shared/extra-utils
parentf88ae8f5bc223579313b28582de9101944a4a814 (diff)
downloadPeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.gz
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.tar.zst
PeerTube-0305db28c98fd6cf43a3c50ba92c76215e99d512.zip
Add support for saving video files to object storage (#4290)
* Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'shared/extra-utils')
-rw-r--r--shared/extra-utils/miscs/checks.ts7
-rw-r--r--shared/extra-utils/miscs/tests.ts15
-rw-r--r--shared/extra-utils/mock-servers/index.ts1
-rw-r--r--shared/extra-utils/mock-servers/mock-object-storage.ts42
-rw-r--r--shared/extra-utils/requests/requests.ts16
-rw-r--r--shared/extra-utils/server/config-command.ts72
-rw-r--r--shared/extra-utils/server/index.ts1
-rw-r--r--shared/extra-utils/server/jobs-command.ts10
-rw-r--r--shared/extra-utils/server/jobs.ts6
-rw-r--r--shared/extra-utils/server/object-storage-command.ts77
-rw-r--r--shared/extra-utils/server/server.ts18
-rw-r--r--shared/extra-utils/server/servers.ts4
-rw-r--r--shared/extra-utils/videos/live-command.ts2
-rw-r--r--shared/extra-utils/videos/live.ts7
-rw-r--r--shared/extra-utils/videos/streaming-playlists-command.ts6
-rw-r--r--shared/extra-utils/videos/streaming-playlists.ts7
-rw-r--r--shared/extra-utils/videos/videos-command.ts11
17 files changed, 286 insertions, 16 deletions
diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts
index 7fc92f804..aa2c8e8fa 100644
--- a/shared/extra-utils/miscs/checks.ts
+++ b/shared/extra-utils/miscs/checks.ts
@@ -16,6 +16,10 @@ function dateIsValid (dateString: string, interval = 300000) {
16 return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval 16 return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval
17} 17}
18 18
19function expectStartWith (str: string, start: string) {
20 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
21}
22
19async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { 23async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
20 const res = await makeGetRequest({ 24 const res = await makeGetRequest({
21 url, 25 url,
@@ -42,5 +46,6 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f
42export { 46export {
43 dateIsValid, 47 dateIsValid,
44 testImage, 48 testImage,
45 testFileExistsOrNot 49 testFileExistsOrNot,
50 expectStartWith
46} 51}
diff --git a/shared/extra-utils/miscs/tests.ts b/shared/extra-utils/miscs/tests.ts
index 3dfb2487e..dd86041fe 100644
--- a/shared/extra-utils/miscs/tests.ts
+++ b/shared/extra-utils/miscs/tests.ts
@@ -28,7 +28,9 @@ const FIXTURE_URLS = {
28 28
29 badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', 29 badVideo: 'https://download.cpy.re/peertube/bad_video.mp4',
30 goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', 30 goodVideo: 'https://download.cpy.re/peertube/good_video.mp4',
31 video4K: 'https://download.cpy.re/peertube/4k_file.txt' 31 goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4',
32
33 file4K: 'https://download.cpy.re/peertube/4k_file.txt'
32} 34}
33 35
34function parallelTests () { 36function parallelTests () {
@@ -42,7 +44,15 @@ function isGithubCI () {
42function areHttpImportTestsDisabled () { 44function areHttpImportTestsDisabled () {
43 const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' 45 const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true'
44 46
45 if (disabled) console.log('Import tests are disabled') 47 if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled')
48
49 return disabled
50}
51
52function areObjectStorageTestsDisabled () {
53 const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true'
54
55 if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled')
46 56
47 return disabled 57 return disabled
48} 58}
@@ -89,6 +99,7 @@ export {
89 buildAbsoluteFixturePath, 99 buildAbsoluteFixturePath,
90 getFileSize, 100 getFileSize,
91 buildRequestStub, 101 buildRequestStub,
102 areObjectStorageTestsDisabled,
92 wait, 103 wait,
93 root 104 root
94} 105}
diff --git a/shared/extra-utils/mock-servers/index.ts b/shared/extra-utils/mock-servers/index.ts
index 0ec07f685..93c00c788 100644
--- a/shared/extra-utils/mock-servers/index.ts
+++ b/shared/extra-utils/mock-servers/index.ts
@@ -2,3 +2,4 @@ export * from './mock-email'
2export * from './mock-instances-index' 2export * from './mock-instances-index'
3export * from './mock-joinpeertube-versions' 3export * from './mock-joinpeertube-versions'
4export * from './mock-plugin-blocklist' 4export * from './mock-plugin-blocklist'
5export * from './mock-object-storage'
diff --git a/shared/extra-utils/mock-servers/mock-object-storage.ts b/shared/extra-utils/mock-servers/mock-object-storage.ts
new file mode 100644
index 000000000..19ea7c87c
--- /dev/null
+++ b/shared/extra-utils/mock-servers/mock-object-storage.ts
@@ -0,0 +1,42 @@
1import * as express from 'express'
2import got, { RequestError } from 'got'
3import { Server } from 'http'
4import { pipeline } from 'stream'
5import { randomInt } from '@shared/core-utils'
6import { ObjectStorageCommand } from '../server'
7
8export class MockObjectStorage {
9 private server: Server
10
11 initialize () {
12 return new Promise<number>(res => {
13 const app = express()
14
15 app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {
16 const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}`
17
18 if (process.env.DEBUG) {
19 console.log('Receiving request on mocked server %s.', req.url)
20 console.log('Proxifying request to %s', url)
21 }
22
23 return pipeline(
24 got.stream(url, { throwHttpErrors: false }),
25 res,
26 (err: RequestError) => {
27 if (!err) return
28
29 console.error('Pipeline failed.', err)
30 }
31 )
32 })
33
34 const port = 42301 + randomInt(1, 100)
35 this.server = app.listen(port, () => res(port))
36 })
37 }
38
39 terminate () {
40 if (this.server) this.server.close()
41 }
42}
diff --git a/shared/extra-utils/requests/requests.ts b/shared/extra-utils/requests/requests.ts
index 70f790222..e3ecd1af2 100644
--- a/shared/extra-utils/requests/requests.ts
+++ b/shared/extra-utils/requests/requests.ts
@@ -121,6 +121,20 @@ function unwrapText (test: request.Test): Promise<string> {
121 return test.then(res => res.text) 121 return test.then(res => res.text)
122} 122}
123 123
124function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> {
125 return test.then(res => {
126 if (res.body instanceof Buffer) {
127 return JSON.parse(new TextDecoder().decode(res.body))
128 }
129
130 return res.body
131 })
132}
133
134function unwrapTextOrDecode (test: request.Test): Promise<string> {
135 return test.then(res => res.text || new TextDecoder().decode(res.body))
136}
137
124// --------------------------------------------------------------------------- 138// ---------------------------------------------------------------------------
125 139
126export { 140export {
@@ -134,6 +148,8 @@ export {
134 makeRawRequest, 148 makeRawRequest,
135 makeActivityPubGetRequest, 149 makeActivityPubGetRequest,
136 unwrapBody, 150 unwrapBody,
151 unwrapTextOrDecode,
152 unwrapBodyOrDecodeToJSON,
137 unwrapText 153 unwrapText
138} 154}
139 155
diff --git a/shared/extra-utils/server/config-command.ts b/shared/extra-utils/server/config-command.ts
index 11148aa46..51d04fa63 100644
--- a/shared/extra-utils/server/config-command.ts
+++ b/shared/extra-utils/server/config-command.ts
@@ -18,6 +18,70 @@ export class ConfigCommand extends AbstractCommand {
18 } 18 }
19 } 19 }
20 20
21 enableImports () {
22 return this.updateExistingSubConfig({
23 newConfig: {
24 import: {
25 videos: {
26 http: {
27 enabled: true
28 },
29
30 torrent: {
31 enabled: true
32 }
33 }
34 }
35 }
36 })
37 }
38
39 enableLive (options: {
40 allowReplay?: boolean
41 transcoding?: boolean
42 } = {}) {
43 return this.updateExistingSubConfig({
44 newConfig: {
45 live: {
46 enabled: true,
47 allowReplay: options.allowReplay ?? true,
48 transcoding: {
49 enabled: options.transcoding ?? true,
50 resolutions: ConfigCommand.getCustomConfigResolutions(true)
51 }
52 }
53 }
54 })
55 }
56
57 disableTranscoding () {
58 return this.updateExistingSubConfig({
59 newConfig: {
60 transcoding: {
61 enabled: false
62 }
63 }
64 })
65 }
66
67 enableTranscoding (webtorrent = true, hls = true) {
68 return this.updateExistingSubConfig({
69 newConfig: {
70 transcoding: {
71 enabled: true,
72 resolutions: ConfigCommand.getCustomConfigResolutions(true),
73
74 webtorrent: {
75 enabled: webtorrent
76 },
77 hls: {
78 enabled: hls
79 }
80 }
81 }
82 })
83 }
84
21 getConfig (options: OverrideCommandOptions = {}) { 85 getConfig (options: OverrideCommandOptions = {}) {
22 const path = '/api/v1/config' 86 const path = '/api/v1/config'
23 87
@@ -81,6 +145,14 @@ export class ConfigCommand extends AbstractCommand {
81 }) 145 })
82 } 146 }
83 147
148 async updateExistingSubConfig (options: OverrideCommandOptions & {
149 newConfig: DeepPartial<CustomConfig>
150 }) {
151 const existing = await this.getCustomConfig(options)
152
153 return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) })
154 }
155
84 updateCustomSubConfig (options: OverrideCommandOptions & { 156 updateCustomSubConfig (options: OverrideCommandOptions & {
85 newConfig: DeepPartial<CustomConfig> 157 newConfig: DeepPartial<CustomConfig>
86 }) { 158 }) {
diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts
index 9055dfc57..92ff7a0f9 100644
--- a/shared/extra-utils/server/index.ts
+++ b/shared/extra-utils/server/index.ts
@@ -6,6 +6,7 @@ export * from './follows-command'
6export * from './follows' 6export * from './follows'
7export * from './jobs' 7export * from './jobs'
8export * from './jobs-command' 8export * from './jobs-command'
9export * from './object-storage-command'
9export * from './plugins-command' 10export * from './plugins-command'
10export * from './plugins' 11export * from './plugins'
11export * from './redundancy-command' 12export * from './redundancy-command'
diff --git a/shared/extra-utils/server/jobs-command.ts b/shared/extra-utils/server/jobs-command.ts
index c4eb12dc2..91771c176 100644
--- a/shared/extra-utils/server/jobs-command.ts
+++ b/shared/extra-utils/server/jobs-command.ts
@@ -5,6 +5,16 @@ import { AbstractCommand, OverrideCommandOptions } from '../shared'
5 5
6export class JobsCommand extends AbstractCommand { 6export class JobsCommand extends AbstractCommand {
7 7
8 async getLatest (options: OverrideCommandOptions & {
9 jobType: JobType
10 }) {
11 const { data } = await this.getJobsList({ ...options, start: 0, count: 1, sort: '-createdAt' })
12
13 if (data.length === 0) return undefined
14
15 return data[0]
16 }
17
8 getJobsList (options: OverrideCommandOptions & { 18 getJobsList (options: OverrideCommandOptions & {
9 state?: JobState 19 state?: JobState
10 jobType?: JobType 20 jobType?: JobType
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts
index 64a0353eb..27104bfdf 100644
--- a/shared/extra-utils/server/jobs.ts
+++ b/shared/extra-utils/server/jobs.ts
@@ -3,7 +3,7 @@ import { JobState } from '../../models'
3import { wait } from '../miscs' 3import { wait } from '../miscs'
4import { PeerTubeServer } from './server' 4import { PeerTubeServer } from './server'
5 5
6async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { 6async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer, skipDelayed = false) {
7 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT 7 const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT
8 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) 8 ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10)
9 : 250 9 : 250
@@ -13,7 +13,9 @@ async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) {
13 if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] 13 if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ]
14 else servers = serversArg as PeerTubeServer[] 14 else servers = serversArg as PeerTubeServer[]
15 15
16 const states: JobState[] = [ 'waiting', 'active', 'delayed' ] 16 const states: JobState[] = [ 'waiting', 'active' ]
17 if (!skipDelayed) states.push('delayed')
18
17 const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] 19 const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ]
18 let pendingRequests: boolean 20 let pendingRequests: boolean
19 21
diff --git a/shared/extra-utils/server/object-storage-command.ts b/shared/extra-utils/server/object-storage-command.ts
new file mode 100644
index 000000000..b4de8f4cb
--- /dev/null
+++ b/shared/extra-utils/server/object-storage-command.ts
@@ -0,0 +1,77 @@
1
2import { HttpStatusCode } from '@shared/models'
3import { makePostBodyRequest } from '../requests'
4import { AbstractCommand } from '../shared'
5
6export class ObjectStorageCommand extends AbstractCommand {
7 static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists'
8 static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos'
9
10 static getDefaultConfig () {
11 return {
12 object_storage: {
13 enabled: true,
14 endpoint: 'http://' + this.getEndpointHost(),
15 region: this.getRegion(),
16
17 credentials: this.getCredentialsConfig(),
18
19 streaming_playlists: {
20 bucket_name: this.DEFAULT_PLAYLIST_BUCKET
21 },
22
23 videos: {
24 bucket_name: this.DEFAULT_WEBTORRENT_BUCKET
25 }
26 }
27 }
28 }
29
30 static getCredentialsConfig () {
31 return {
32 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
33 secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
34 }
35 }
36
37 static getEndpointHost () {
38 return 'localhost:9444'
39 }
40
41 static getRegion () {
42 return 'us-east-1'
43 }
44
45 static getWebTorrentBaseUrl () {
46 return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/`
47 }
48
49 static getPlaylistBaseUrl () {
50 return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/`
51 }
52
53 static async prepareDefaultBuckets () {
54 await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET)
55 await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET)
56 }
57
58 static async createBucket (name: string) {
59 await makePostBodyRequest({
60 url: this.getEndpointHost(),
61 path: '/ui/' + name + '?delete',
62 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
63 })
64
65 await makePostBodyRequest({
66 url: this.getEndpointHost(),
67 path: '/ui/' + name + '?create',
68 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
69 })
70
71 await makePostBodyRequest({
72 url: this.getEndpointHost(),
73 path: '/ui/' + name + '?make-public',
74 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
75 })
76 }
77}
diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts
index 3c335b8e4..bc5e1cd5f 100644
--- a/shared/extra-utils/server/server.ts
+++ b/shared/extra-utils/server/server.ts
@@ -38,11 +38,13 @@ import { PluginsCommand } from './plugins-command'
38import { RedundancyCommand } from './redundancy-command' 38import { RedundancyCommand } from './redundancy-command'
39import { ServersCommand } from './servers-command' 39import { ServersCommand } from './servers-command'
40import { StatsCommand } from './stats-command' 40import { StatsCommand } from './stats-command'
41import { ObjectStorageCommand } from './object-storage-command'
41 42
42export type RunServerOptions = { 43export type RunServerOptions = {
43 hideLogs?: boolean 44 hideLogs?: boolean
44 nodeArgs?: string[] 45 nodeArgs?: string[]
45 peertubeArgs?: string[] 46 peertubeArgs?: string[]
47 env?: { [ id: string ]: string }
46} 48}
47 49
48export class PeerTubeServer { 50export class PeerTubeServer {
@@ -121,6 +123,7 @@ export class PeerTubeServer {
121 servers?: ServersCommand 123 servers?: ServersCommand
122 login?: LoginCommand 124 login?: LoginCommand
123 users?: UsersCommand 125 users?: UsersCommand
126 objectStorage?: ObjectStorageCommand
124 videos?: VideosCommand 127 videos?: VideosCommand
125 128
126 constructor (options: { serverNumber: number } | { url: string }) { 129 constructor (options: { serverNumber: number } | { url: string }) {
@@ -202,6 +205,10 @@ export class PeerTubeServer {
202 env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() 205 env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
203 env['NODE_CONFIG'] = JSON.stringify(configOverride) 206 env['NODE_CONFIG'] = JSON.stringify(configOverride)
204 207
208 if (options.env) {
209 Object.assign(env, options.env)
210 }
211
205 const forkOptions = { 212 const forkOptions = {
206 silent: true, 213 silent: true,
207 env, 214 env,
@@ -209,10 +216,17 @@ export class PeerTubeServer {
209 execArgv: options.nodeArgs || [] 216 execArgv: options.nodeArgs || []
210 } 217 }
211 218
212 return new Promise<void>(res => { 219 return new Promise<void>((res, rej) => {
213 const self = this 220 const self = this
214 221
215 this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions) 222 this.app = fork(join(root(), 'dist', 'server.js'), options.peertubeArgs || [], forkOptions)
223
224 const onExit = function () {
225 return rej(new Error('Process exited'))
226 }
227
228 this.app.on('exit', onExit)
229
216 this.app.stdout.on('data', function onStdout (data) { 230 this.app.stdout.on('data', function onStdout (data) {
217 let dontContinue = false 231 let dontContinue = false
218 232
@@ -241,6 +255,7 @@ export class PeerTubeServer {
241 console.log(data.toString()) 255 console.log(data.toString())
242 } else { 256 } else {
243 self.app.stdout.removeListener('data', onStdout) 257 self.app.stdout.removeListener('data', onStdout)
258 self.app.removeListener('exit', onExit)
244 } 259 }
245 260
246 process.on('exit', () => { 261 process.on('exit', () => {
@@ -365,5 +380,6 @@ export class PeerTubeServer {
365 this.login = new LoginCommand(this) 380 this.login = new LoginCommand(this)
366 this.users = new UsersCommand(this) 381 this.users = new UsersCommand(this)
367 this.videos = new VideosCommand(this) 382 this.videos = new VideosCommand(this)
383 this.objectStorage = new ObjectStorageCommand(this)
368 } 384 }
369} 385}
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts
index f0622feb0..21ab9405b 100644
--- a/shared/extra-utils/server/servers.ts
+++ b/shared/extra-utils/server/servers.ts
@@ -10,11 +10,11 @@ async function createSingleServer (serverNumber: number, configOverride?: Object
10 return server 10 return server
11} 11}
12 12
13function createMultipleServers (totalServers: number, configOverride?: Object) { 13function createMultipleServers (totalServers: number, configOverride?: Object, options: RunServerOptions = {}) {
14 const serverPromises: Promise<PeerTubeServer>[] = [] 14 const serverPromises: Promise<PeerTubeServer>[] = []
15 15
16 for (let i = 1; i <= totalServers; i++) { 16 for (let i = 1; i <= totalServers; i++) {
17 serverPromises.push(createSingleServer(i, configOverride)) 17 serverPromises.push(createSingleServer(i, configOverride, options))
18 } 18 }
19 19
20 return Promise.all(serverPromises) 20 return Promise.all(serverPromises)
diff --git a/shared/extra-utils/videos/live-command.ts b/shared/extra-utils/videos/live-command.ts
index 81ae458e0..74f5d3089 100644
--- a/shared/extra-utils/videos/live-command.ts
+++ b/shared/extra-utils/videos/live-command.ts
@@ -126,7 +126,7 @@ export class LiveCommand extends AbstractCommand {
126 video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) 126 video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId })
127 127
128 await wait(500) 128 await wait(500)
129 } while (video.isLive === true && video.state.id !== VideoState.PUBLISHED) 129 } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED)
130 } 130 }
131 131
132 async countPlaylists (options: OverrideCommandOptions & { 132 async countPlaylists (options: OverrideCommandOptions & {
diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts
index 9a6df07a8..29f99ed6d 100644
--- a/shared/extra-utils/videos/live.ts
+++ b/shared/extra-utils/videos/live.ts
@@ -89,6 +89,12 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi
89 } 89 }
90} 90}
91 91
92async function waitUntilLiveSavedOnAllServers (servers: PeerTubeServer[], videoId: string) {
93 for (const server of servers) {
94 await server.live.waitUntilSaved({ videoId })
95 }
96}
97
92async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { 98async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
93 const basePath = server.servers.buildDirectory('streaming-playlists') 99 const basePath = server.servers.buildDirectory('streaming-playlists')
94 const hlsPath = join(basePath, 'hls', videoUUID) 100 const hlsPath = join(basePath, 'hls', videoUUID)
@@ -126,5 +132,6 @@ export {
126 testFfmpegStreamError, 132 testFfmpegStreamError,
127 stopFfmpeg, 133 stopFfmpeg,
128 waitUntilLivePublishedOnAllServers, 134 waitUntilLivePublishedOnAllServers,
135 waitUntilLiveSavedOnAllServers,
129 checkLiveCleanupAfterSave 136 checkLiveCleanupAfterSave
130} 137}
diff --git a/shared/extra-utils/videos/streaming-playlists-command.ts b/shared/extra-utils/videos/streaming-playlists-command.ts
index 9662685da..5d40d35cb 100644
--- a/shared/extra-utils/videos/streaming-playlists-command.ts
+++ b/shared/extra-utils/videos/streaming-playlists-command.ts
@@ -1,5 +1,5 @@
1import { HttpStatusCode } from '@shared/models' 1import { HttpStatusCode } from '@shared/models'
2import { unwrapBody, unwrapText } from '../requests' 2import { unwrapBody, unwrapTextOrDecode, unwrapBodyOrDecodeToJSON } from '../requests'
3import { AbstractCommand, OverrideCommandOptions } from '../shared' 3import { AbstractCommand, OverrideCommandOptions } from '../shared'
4 4
5export class StreamingPlaylistsCommand extends AbstractCommand { 5export class StreamingPlaylistsCommand extends AbstractCommand {
@@ -7,7 +7,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
7 get (options: OverrideCommandOptions & { 7 get (options: OverrideCommandOptions & {
8 url: string 8 url: string
9 }) { 9 }) {
10 return unwrapText(this.getRawRequest({ 10 return unwrapTextOrDecode(this.getRawRequest({
11 ...options, 11 ...options,
12 12
13 url: options.url, 13 url: options.url,
@@ -33,7 +33,7 @@ export class StreamingPlaylistsCommand extends AbstractCommand {
33 getSegmentSha256 (options: OverrideCommandOptions & { 33 getSegmentSha256 (options: OverrideCommandOptions & {
34 url: string 34 url: string
35 }) { 35 }) {
36 return unwrapBody<{ [ id: string ]: string }>(this.getRawRequest({ 36 return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({
37 ...options, 37 ...options,
38 38
39 url: options.url, 39 url: options.url,
diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts
index a224b8f5f..6671e3fa6 100644
--- a/shared/extra-utils/videos/streaming-playlists.ts
+++ b/shared/extra-utils/videos/streaming-playlists.ts
@@ -9,17 +9,16 @@ async function checkSegmentHash (options: {
9 server: PeerTubeServer 9 server: PeerTubeServer
10 baseUrlPlaylist: string 10 baseUrlPlaylist: string
11 baseUrlSegment: string 11 baseUrlSegment: string
12 videoUUID: string
13 resolution: number 12 resolution: number
14 hlsPlaylist: VideoStreamingPlaylist 13 hlsPlaylist: VideoStreamingPlaylist
15}) { 14}) {
16 const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options 15 const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist } = options
17 const command = server.streamingPlaylists 16 const command = server.streamingPlaylists
18 17
19 const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) 18 const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
20 const videoName = basename(file.fileUrl) 19 const videoName = basename(file.fileUrl)
21 20
22 const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` }) 21 const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8` })
23 22
24 const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) 23 const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
25 24
@@ -28,7 +27,7 @@ async function checkSegmentHash (options: {
28 const range = `${offset}-${offset + length - 1}` 27 const range = `${offset}-${offset + length - 1}`
29 28
30 const segmentBody = await command.getSegment({ 29 const segmentBody = await command.getSegment({
31 url: `${baseUrlSegment}/${videoUUID}/${videoName}`, 30 url: `${baseUrlSegment}/${videoName}`,
32 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, 31 expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206,
33 range: `bytes=${range}` 32 range: `bytes=${range}`
34 }) 33 })
diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts
index 33725bfdc..d35339c8d 100644
--- a/shared/extra-utils/videos/videos-command.ts
+++ b/shared/extra-utils/videos/videos-command.ts
@@ -188,6 +188,17 @@ export class VideosCommand extends AbstractCommand {
188 return id 188 return id
189 } 189 }
190 190
191 async listFiles (options: OverrideCommandOptions & {
192 id: number | string
193 }) {
194 const video = await this.get(options)
195
196 const files = video.files || []
197 const hlsFiles = video.streamingPlaylists[0]?.files || []
198
199 return files.concat(hlsFiles)
200 }
201
191 // --------------------------------------------------------------------------- 202 // ---------------------------------------------------------------------------
192 203
193 listMyVideos (options: OverrideCommandOptions & { 204 listMyVideos (options: OverrideCommandOptions & {