From f89189907bbdff6c4bc6d3460ed9ef4c49515f17 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 23 May 2023 10:49:45 +0200 Subject: Prevent object storage mock conflicts When running tests in parallel --- server/tests/api/object-storage/live.ts | 58 +++++++++++++++--------- server/tests/api/object-storage/video-imports.ts | 13 ++++-- server/tests/api/object-storage/videos.ts | 50 ++++++++------------ 3 files changed, 65 insertions(+), 56 deletions(-) (limited to 'server/tests/api/object-storage') diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts index 7f8db40e5..07ff4763b 100644 --- a/server/tests/api/object-storage/live.ts +++ b/server/tests/api/object-storage/live.ts @@ -36,7 +36,14 @@ async function createLive (server: PeerTubeServer, permanent: boolean) { return uuid } -async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) { +async function checkFilesExist (options: { + servers: PeerTubeServer[] + videoUUID: string + numberOfFiles: number + objectStorage: ObjectStorageCommand +}) { + const { servers, videoUUID, numberOfFiles, objectStorage } = options + for (const server of servers) { const video = await server.videos.get({ id: videoUUID }) @@ -47,14 +54,21 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu expect(files).to.have.lengthOf(numberOfFiles) for (const file of files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } } } -async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) { +async function checkFilesCleanup (options: { + server: PeerTubeServer + videoUUID: string + resolutions: number[] + objectStorage: ObjectStorageCommand +}) { + const { server, videoUUID, resolutions, objectStorage } = options + const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`) for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) { @@ -62,7 +76,7 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res videoUUID, playlistName, expectedStatus: HttpStatusCode.NOT_FOUND_404, - objectStorage: true + objectStorage }) } @@ -70,7 +84,7 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res videoUUID, playlistNumber: 0, segment: 0, - objectStorage: true, + objectStorage, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) } @@ -80,13 +94,13 @@ describe('Object storage for lives', function () { let servers: PeerTubeServer[] let sqlCommandServer1: SQLCommand + const objectStorage = new ObjectStorageCommand() before(async function () { this.timeout(120000) - await ObjectStorageCommand.prepareDefaultMockBuckets() - - servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig()) + await objectStorage.prepareDefaultMockBuckets() + servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig()) await setAccessTokensToServers(servers) await setDefaultVideoChannel(servers) @@ -119,7 +133,7 @@ describe('Object storage for lives', function () { liveVideoId: videoUUID, resolutions: [ 720 ], transcoded: false, - objectStorage: true + objectStorage }) await stopFfmpeg(ffmpegCommand) @@ -131,11 +145,11 @@ describe('Object storage for lives', function () { await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID) await waitJobs(servers) - await checkFilesExist(servers, videoUUID, 1) + await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage }) }) it('Should have cleaned up live files from object storage', async function () { - await checkFilesCleanup(servers[0], videoUUID, [ 720 ]) + await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage }) }) }) @@ -166,7 +180,7 @@ describe('Object storage for lives', function () { liveVideoId: videoUUIDNonPermanent, resolutions, transcoded: true, - objectStorage: true + objectStorage }) await stopFfmpeg(ffmpegCommand) @@ -178,11 +192,11 @@ describe('Object storage for lives', function () { await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent) await waitJobs(servers) - await checkFilesExist(servers, videoUUIDNonPermanent, 5) + await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage }) }) it('Should have cleaned up live files from object storage', async function () { - await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions) + await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage }) }) }) @@ -206,7 +220,7 @@ describe('Object storage for lives', function () { liveVideoId: videoUUIDPermanent, resolutions, transcoded: true, - objectStorage: true + objectStorage }) await stopFfmpeg(ffmpegCommand) @@ -221,11 +235,11 @@ describe('Object storage for lives', function () { const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent }) const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) - await checkFilesExist(servers, replay.uuid, 5) + await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage }) }) it('Should have cleaned up live files from object storage', async function () { - await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions) + await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage }) }) }) }) @@ -238,9 +252,10 @@ describe('Object storage for lives', function () { this.timeout(120000) const port = await mockObjectStorageProxy.initialize() - baseMockUrl = `http://127.0.0.1:${port}/streaming-playlists` + const bucketName = objectStorage.getMockStreamingPlaylistsBucketName() + baseMockUrl = `http://127.0.0.1:${port}/${bucketName}` - await ObjectStorageCommand.createMockBucket('streaming-playlists') + await objectStorage.prepareDefaultMockBuckets() const config = { object_storage: { @@ -251,7 +266,7 @@ describe('Object storage for lives', function () { credentials: ObjectStorageCommand.getMockCredentialsConfig(), streaming_playlists: { - bucket_name: 'streaming-playlists', + bucket_name: bucketName, prefix: '', base_url: baseMockUrl } @@ -279,7 +294,7 @@ describe('Object storage for lives', function () { liveVideoId: videoUUIDPermanent, resolutions: [ 720 ], transcoded: true, - objectStorage: true, + objectStorage, objectStorageBaseUrl: baseMockUrl }) @@ -289,6 +304,7 @@ describe('Object storage for lives', function () { after(async function () { await sqlCommandServer1.cleanup() + await objectStorage.cleanupMock() await cleanupTests(servers) }) diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts index d5fd91b6a..57150e5a6 100644 --- a/server/tests/api/object-storage/video-imports.ts +++ b/server/tests/api/object-storage/video-imports.ts @@ -32,13 +32,14 @@ describe('Object storage for video import', function () { if (areMockObjectStorageTestsDisabled()) return let server: PeerTubeServer + const objectStorage = new ObjectStorageCommand() before(async function () { this.timeout(120000) - await ObjectStorageCommand.prepareDefaultMockBuckets() + await objectStorage.prepareDefaultMockBuckets() - server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig()) + server = await createSingleServer(1, objectStorage.getDefaultMockConfig()) await setAccessTokensToServers([ server ]) await setDefaultVideoChannel([ server ]) @@ -64,7 +65,7 @@ describe('Object storage for video import', function () { expect(video.streamingPlaylists).to.have.lengthOf(0) const fileUrl = video.files[0].fileUrl - expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) + expectStartWith(fileUrl, objectStorage.getMockWebVideosBaseUrl()) await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) }) @@ -89,13 +90,13 @@ describe('Object storage for video import', function () { expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) for (const file of video.files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } for (const file of video.streamingPlaylists[0].files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } @@ -103,6 +104,8 @@ describe('Object storage for video import', function () { }) after(async function () { + await objectStorage.cleanupMock() + await cleanupTests([ server ]) }) }) diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts index 77ed78ae1..f837d9966 100644 --- a/server/tests/api/object-storage/videos.ts +++ b/server/tests/api/object-storage/videos.ts @@ -145,6 +145,7 @@ function runTestSuite (options: { let servers: PeerTubeServer[] let sqlCommands: SQLCommand[] = [] + const objectStorage = new ObjectStorageCommand() let keptUrls: string[] = [] @@ -159,8 +160,8 @@ function runTestSuite (options: { ? `http://127.0.0.1:${port}` : undefined - await ObjectStorageCommand.createMockBucket(options.playlistBucket) - await ObjectStorageCommand.createMockBucket(options.webtorrentBucket) + await objectStorage.createMockBucket(options.playlistBucket) + await objectStorage.createMockBucket(options.webtorrentBucket) const config = { object_storage: { @@ -275,6 +276,7 @@ function runTestSuite (options: { after(async function () { await mockObjectStorageProxy.terminate() + await objectStorage.cleanupMock() for (const sqlCommand of sqlCommands) { await sqlCommand.cleanup() @@ -287,26 +289,12 @@ function runTestSuite (options: { describe('Object storage for videos', function () { if (areMockObjectStorageTestsDisabled()) return + const objectStorage = new ObjectStorageCommand() + describe('Test config', function () { let server: PeerTubeServer - const baseConfig = { - object_storage: { - enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), - region: ObjectStorageCommand.getMockRegion(), - - credentials: ObjectStorageCommand.getMockCredentialsConfig(), - - streaming_playlists: { - bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET - }, - - videos: { - bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET - } - } - } + const baseConfig = objectStorage.getDefaultMockConfig() const badCredentials = { access_key_id: 'AKIAIOSFODNN7EXAMPLE', @@ -334,7 +322,7 @@ describe('Object storage for videos', function () { it('Should fail with bad credentials', async function () { this.timeout(60000) - await ObjectStorageCommand.prepareDefaultMockBuckets() + await objectStorage.prepareDefaultMockBuckets() const config = merge({}, baseConfig, { object_storage: { @@ -358,7 +346,7 @@ describe('Object storage for videos', function () { it('Should succeed with credentials from env', async function () { this.timeout(60000) - await ObjectStorageCommand.prepareDefaultMockBuckets() + await objectStorage.prepareDefaultMockBuckets() const config = merge({}, baseConfig, { object_storage: { @@ -385,25 +373,27 @@ describe('Object storage for videos', function () { await waitJobs([ server ], { skipDelayed: true }) const video = await server.videos.get({ id: uuid }) - expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) + expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) }) after(async function () { + await objectStorage.cleanupMock() + await cleanupTests([ server ]) }) }) describe('Test simple object storage', function () { runTestSuite({ - playlistBucket: 'streaming-playlists', - webtorrentBucket: 'videos' + playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), + webtorrentBucket: objectStorage.getMockBucketName('videos') }) }) describe('Test object storage with prefix', function () { runTestSuite({ - playlistBucket: 'mybucket', - webtorrentBucket: 'mybucket', + playlistBucket: objectStorage.getMockBucketName('mybucket'), + webtorrentBucket: objectStorage.getMockBucketName('mybucket'), playlistPrefix: 'streaming-playlists_', webtorrentPrefix: 'webtorrent_' @@ -412,8 +402,8 @@ describe('Object storage for videos', function () { describe('Test object storage with prefix and base URL', function () { runTestSuite({ - playlistBucket: 'mybucket', - webtorrentBucket: 'mybucket', + playlistBucket: objectStorage.getMockBucketName('mybucket'), + webtorrentBucket: objectStorage.getMockBucketName('mybucket'), playlistPrefix: 'streaming-playlists/', webtorrentPrefix: 'webtorrent/', @@ -440,8 +430,8 @@ describe('Object storage for videos', function () { runTestSuite({ maxUploadPart, - playlistBucket: 'streaming-playlists', - webtorrentBucket: 'videos', + playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), + webtorrentBucket: objectStorage.getMockBucketName('videos'), fixture }) }) -- cgit v1.2.3