From 8059e050ef4e800c95851ac97abbb2330fe17882 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 27 Jan 2023 08:30:56 +0100 Subject: Fix live with base url object storage --- server/lib/live/live-manager.ts | 4 +- server/lib/object-storage/urls.ts | 2 + server/tests/api/object-storage/live.ts | 62 ++++++++++++++++++++-- server/tests/api/object-storage/videos.ts | 12 +++-- server/tests/shared/live.ts | 23 +++++--- .../shared/mock-servers/mock-object-storage.ts | 2 +- shared/server-commands/videos/live-command.ts | 12 ++++- 7 files changed, 99 insertions(+), 18 deletions(-) diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts index 9ea983119..fa4e1df07 100644 --- a/server/lib/live/live-manager.ts +++ b/server/lib/live/live-manager.ts @@ -487,7 +487,9 @@ class LiveManager { ? VideoStorage.OBJECT_STORAGE : VideoStorage.FILE_SYSTEM - playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) + if (playlist.storage === VideoStorage.FILE_SYSTEM) { + playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions) + } return playlist.save() } diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts index a47a98b98..b8ef94559 100644 --- a/server/lib/object-storage/urls.ts +++ b/server/lib/object-storage/urls.ts @@ -57,5 +57,7 @@ function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { const regex = new RegExp('https?://[^/]+') function replaceByBaseUrl (fileUrl: string, baseUrl: string) { + if (!fileUrl) return fileUrl + return fileUrl.replace(regex, baseUrl) } diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts index ad2b554b7..2a3fc4779 100644 --- a/server/tests/api/object-storage/live.ts +++ b/server/tests/api/object-storage/live.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { expectStartWith, testVideoResolutions } from '@server/tests/shared' +import { expectStartWith, MockObjectStorageProxy, testVideoResolutions } from '@server/tests/shared' import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' import { @@ -93,7 +93,7 @@ describe('Object storage for lives', function () { await servers[0].config.enableTranscoding() }) - describe('Without live transcoding', async function () { + describe('Without live transcoding', function () { let videoUUID: string before(async function () { @@ -134,7 +134,7 @@ describe('Object storage for lives', function () { }) }) - describe('With live transcoding', async function () { + describe('With live transcoding', function () { const resolutions = [ 720, 480, 360, 240, 144 ] before(async function () { @@ -223,6 +223,62 @@ describe('Object storage for lives', function () { }) }) + describe('With object storage base url', function () { + const mockObjectStorageProxy = new MockObjectStorageProxy() + let baseMockUrl: string + + before(async function () { + this.timeout(120000) + + const port = await mockObjectStorageProxy.initialize() + baseMockUrl = `http://127.0.0.1:${port}/streaming-playlists` + + await ObjectStorageCommand.createMockBucket('streaming-playlists') + + const config = { + object_storage: { + enabled: true, + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), + + credentials: ObjectStorageCommand.getMockCredentialsConfig(), + + streaming_playlists: { + bucket_name: 'streaming-playlists', + prefix: '', + base_url: baseMockUrl + } + } + } + + await servers[0].kill() + await servers[0].run(config) + + await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' }) + }) + + it('Should publish a live and replace the base url', async function () { + this.timeout(240000) + + const videoUUIDPermanent = await createLive(servers[0], true) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) + await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) + + await testVideoResolutions({ + originServer: servers[0], + servers, + liveVideoId: videoUUIDPermanent, + resolutions: [ 720 ], + transcoded: true, + objectStorage: true, + objectStorageBaseUrl: baseMockUrl + }) + + await stopFfmpeg(ffmpegCommand) + }) + }) + after(async function () { await killallServers(servers) }) diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts index 1b3c389d7..6862658cc 100644 --- a/server/tests/api/object-storage/videos.ts +++ b/server/tests/api/object-storage/videos.ts @@ -9,7 +9,7 @@ import { expectLogDoesNotContain, expectStartWith, generateHighBitrateVideo, - MockObjectStorage + MockObjectStorageProxy } from '@server/tests/shared' import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoDetails } from '@shared/models' @@ -124,7 +124,7 @@ function runTestSuite (options: { useMockBaseUrl?: boolean }) { - const mockObjectStorage = new MockObjectStorage() + const mockObjectStorageProxy = new MockObjectStorageProxy() const { fixture } = options let baseMockUrl: string @@ -138,8 +138,10 @@ function runTestSuite (options: { before(async function () { this.timeout(120000) - const port = await mockObjectStorage.initialize() - baseMockUrl = options.useMockBaseUrl ? `http://127.0.0.1:${port}` : undefined + const port = await mockObjectStorageProxy.initialize() + baseMockUrl = options.useMockBaseUrl + ? `http://127.0.0.1:${port}` + : undefined await ObjectStorageCommand.createMockBucket(options.playlistBucket) await ObjectStorageCommand.createMockBucket(options.webtorrentBucket) @@ -254,7 +256,7 @@ function runTestSuite (options: { }) after(async function () { - await mockObjectStorage.terminate() + await mockObjectStorageProxy.terminate() await cleanupTests(servers) }) diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts index 1c868eb5b..ff0b2f226 100644 --- a/server/tests/shared/live.ts +++ b/server/tests/shared/live.ts @@ -3,10 +3,10 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' +import { sha1 } from '@shared/extra-utils' import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models' import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands' import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists' -import { sha1 } from '@shared/extra-utils' async function checkLiveCleanup (options: { server: PeerTubeServer @@ -42,9 +42,19 @@ async function testVideoResolutions (options: { liveVideoId: string resolutions: number[] transcoded: boolean + objectStorage: boolean + objectStorageBaseUrl?: string }) { - const { originServer, servers, liveVideoId, resolutions, transcoded, objectStorage } = options + const { + originServer, + servers, + liveVideoId, + resolutions, + transcoded, + objectStorage, + objectStorageBaseUrl = ObjectStorageCommand.getMockPlaylistBaseUrl() + } = options for (const server of servers) { const { data } = await server.videos.list() @@ -66,7 +76,7 @@ async function testVideoResolutions (options: { }) if (objectStorage) { - expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl()) + expect(hlsPlaylist.playlistUrl).to.contain(objectStorageBaseUrl) } for (let i = 0; i < resolutions.length; i++) { @@ -77,15 +87,16 @@ async function testVideoResolutions (options: { videoUUID: video.uuid, playlistNumber: i, segment: segmentNum, - objectStorage + objectStorage, + objectStorageBaseUrl }) const baseUrl = objectStorage - ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls' + ? join(objectStorageBaseUrl, 'hls') : originServer.url + '/static/streaming-playlists/hls' if (objectStorage) { - expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl()) + expect(hlsPlaylist.segmentsSha256Url).to.contain(objectStorageBaseUrl) } const subPlaylist = await originServer.streamingPlaylists.get({ diff --git a/server/tests/shared/mock-servers/mock-object-storage.ts b/server/tests/shared/mock-servers/mock-object-storage.ts index 8c325bf11..ae76c4f3f 100644 --- a/server/tests/shared/mock-servers/mock-object-storage.ts +++ b/server/tests/shared/mock-servers/mock-object-storage.ts @@ -5,7 +5,7 @@ import { pipeline } from 'stream' import { ObjectStorageCommand } from '@shared/server-commands' import { getPort, randomListen, terminateServer } from './shared' -export class MockObjectStorage { +export class MockObjectStorageProxy { private server: Server async initialize () { diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts index cc9502c6f..4541465ba 100644 --- a/shared/server-commands/videos/live-command.ts +++ b/shared/server-commands/videos/live-command.ts @@ -192,12 +192,20 @@ export class LiveCommand extends AbstractCommand { playlistNumber: number segment: number objectStorage: boolean + objectStorageBaseUrl?: string }) { - const { server, objectStorage, playlistNumber, segment, videoUUID } = options + const { + server, + objectStorage, + playlistNumber, + segment, + videoUUID, + objectStorageBaseUrl = ObjectStorageCommand.getMockPlaylistBaseUrl() + } = options const segmentName = `${playlistNumber}-00000${segment}.ts` const baseUrl = objectStorage - ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls' + ? join(objectStorageBaseUrl, 'hls') : server.url + '/static/streaming-playlists/hls' let error = true -- cgit v1.2.3