server,
videoUUID: liveVideoUUID,
segment: 1,
- playlistNumber: 0,
- objectStorage: false
+ playlistNumber: 0
})
return { ffmpegCommand, liveVideoUUID }
servers,
liveVideoId,
resolutions: [ 720 ],
- objectStorage: false,
transcoded: true
})
servers,
liveVideoId,
resolutions: resolutions.concat([ 720 ]),
- objectStorage: false,
transcoded: true
})
servers,
liveVideoId,
resolutions,
- objectStorage: false,
transcoded: true
})
servers,
liveVideoId,
resolutions,
- objectStorage: false,
transcoded: true
})
servers,
liveVideoId,
resolutions: [ 720 ],
- objectStorage: false,
transcoded: true
})
server: servers[0],
videoUUID,
playlistNumber: 0,
- segment: 2,
- objectStorage: false
+ segment: 2
})
}
return uuid
}
-async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, numberOfFiles: number) {
+async function checkFilesExist (options: {
+ servers: PeerTubeServer[]
+ videoUUID: string
+ numberOfFiles: number
+ objectStorage: ObjectStorageCommand
+}) {
+ const { servers, videoUUID, numberOfFiles, objectStorage } = options
+
for (const server of servers) {
const video = await server.videos.get({ id: videoUUID })
expect(files).to.have.lengthOf(numberOfFiles)
for (const file of files) {
- expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
}
-async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[]) {
+async function checkFilesCleanup (options: {
+ server: PeerTubeServer
+ videoUUID: string
+ resolutions: number[]
+ objectStorage: ObjectStorageCommand
+}) {
+ const { server, videoUUID, resolutions, objectStorage } = options
+
const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`)
for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) {
videoUUID,
playlistName,
expectedStatus: HttpStatusCode.NOT_FOUND_404,
- objectStorage: true
+ objectStorage
})
}
videoUUID,
playlistNumber: 0,
segment: 0,
- objectStorage: true,
+ objectStorage,
expectedStatus: HttpStatusCode.NOT_FOUND_404
})
}
let servers: PeerTubeServer[]
let sqlCommandServer1: SQLCommand
+ const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
-
- servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig())
+ await objectStorage.prepareDefaultMockBuckets()
+ servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig())
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
liveVideoId: videoUUID,
resolutions: [ 720 ],
transcoded: false,
- objectStorage: true
+ objectStorage
})
await stopFfmpeg(ffmpegCommand)
await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID)
await waitJobs(servers)
- await checkFilesExist(servers, videoUUID, 1)
+ await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage })
})
it('Should have cleaned up live files from object storage', async function () {
- await checkFilesCleanup(servers[0], videoUUID, [ 720 ])
+ await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage })
})
})
liveVideoId: videoUUIDNonPermanent,
resolutions,
transcoded: true,
- objectStorage: true
+ objectStorage
})
await stopFfmpeg(ffmpegCommand)
await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent)
await waitJobs(servers)
- await checkFilesExist(servers, videoUUIDNonPermanent, 5)
+ await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage })
})
it('Should have cleaned up live files from object storage', async function () {
- await checkFilesCleanup(servers[0], videoUUIDNonPermanent, resolutions)
+ await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage })
})
})
liveVideoId: videoUUIDPermanent,
resolutions,
transcoded: true,
- objectStorage: true
+ objectStorage
})
await stopFfmpeg(ffmpegCommand)
const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent })
const replay = await findExternalSavedVideo(servers[0], videoLiveDetails)
- await checkFilesExist(servers, replay.uuid, 5)
+ await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage })
})
it('Should have cleaned up live files from object storage', async function () {
- await checkFilesCleanup(servers[0], videoUUIDPermanent, resolutions)
+ await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage })
})
})
})
this.timeout(120000)
const port = await mockObjectStorageProxy.initialize()
- baseMockUrl = `http://127.0.0.1:${port}/streaming-playlists`
+ const bucketName = objectStorage.getMockStreamingPlaylistsBucketName()
+ baseMockUrl = `http://127.0.0.1:${port}/${bucketName}`
- await ObjectStorageCommand.createMockBucket('streaming-playlists')
+ await objectStorage.prepareDefaultMockBuckets()
const config = {
object_storage: {
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
streaming_playlists: {
- bucket_name: 'streaming-playlists',
+ bucket_name: bucketName,
prefix: '',
base_url: baseMockUrl
}
liveVideoId: videoUUIDPermanent,
resolutions: [ 720 ],
transcoded: true,
- objectStorage: true,
+ objectStorage,
objectStorageBaseUrl: baseMockUrl
})
after(async function () {
await sqlCommandServer1.cleanup()
+ await objectStorage.cleanupMock()
await cleanupTests(servers)
})
if (areMockObjectStorageTestsDisabled()) return
let server: PeerTubeServer
+ const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
- server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig())
+ server = await createSingleServer(1, objectStorage.getDefaultMockConfig())
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
expect(video.streamingPlaylists).to.have.lengthOf(0)
const fileUrl = video.files[0].fileUrl
- expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
})
expect(video.streamingPlaylists[0].files).to.have.lengthOf(5)
for (const file of video.files) {
- expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of video.streamingPlaylists[0].files) {
- expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
})
after(async function () {
+ await objectStorage.cleanupMock()
+
await cleanupTests([ server ])
})
})
let servers: PeerTubeServer[]
let sqlCommands: SQLCommand[] = []
+ const objectStorage = new ObjectStorageCommand()
let keptUrls: string[] = []
? `http://127.0.0.1:${port}`
: undefined
- await ObjectStorageCommand.createMockBucket(options.playlistBucket)
- await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
+ await objectStorage.createMockBucket(options.playlistBucket)
+ await objectStorage.createMockBucket(options.webtorrentBucket)
const config = {
object_storage: {
after(async function () {
await mockObjectStorageProxy.terminate()
+ await objectStorage.cleanupMock()
for (const sqlCommand of sqlCommands) {
await sqlCommand.cleanup()
describe('Object storage for videos', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
describe('Test config', function () {
let server: PeerTubeServer
- const baseConfig = {
- object_storage: {
- enabled: true,
- endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
- region: ObjectStorageCommand.getMockRegion(),
-
- credentials: ObjectStorageCommand.getMockCredentialsConfig(),
-
- streaming_playlists: {
- bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
- },
-
- videos: {
- bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
- }
- }
- }
+ const baseConfig = objectStorage.getDefaultMockConfig()
const badCredentials = {
access_key_id: 'AKIAIOSFODNN7EXAMPLE',
it('Should fail with bad credentials', async function () {
this.timeout(60000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
it('Should succeed with credentials from env', async function () {
this.timeout(60000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
await waitJobs([ server ], { skipDelayed: true })
const video = await server.videos.get({ id: uuid })
- expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
})
after(async function () {
+ await objectStorage.cleanupMock()
+
await cleanupTests([ server ])
})
})
describe('Test simple object storage', function () {
runTestSuite({
- playlistBucket: 'streaming-playlists',
- webtorrentBucket: 'videos'
+ playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
+ webtorrentBucket: objectStorage.getMockBucketName('videos')
})
})
describe('Test object storage with prefix', function () {
runTestSuite({
- playlistBucket: 'mybucket',
- webtorrentBucket: 'mybucket',
+ playlistBucket: objectStorage.getMockBucketName('mybucket'),
+ webtorrentBucket: objectStorage.getMockBucketName('mybucket'),
playlistPrefix: 'streaming-playlists_',
webtorrentPrefix: 'webtorrent_'
describe('Test object storage with prefix and base URL', function () {
runTestSuite({
- playlistBucket: 'mybucket',
- webtorrentBucket: 'mybucket',
+ playlistBucket: objectStorage.getMockBucketName('mybucket'),
+ webtorrentBucket: objectStorage.getMockBucketName('mybucket'),
playlistPrefix: 'streaming-playlists/',
webtorrentPrefix: 'webtorrent/',
runTestSuite({
maxUploadPart,
- playlistBucket: 'streaming-playlists',
- webtorrentBucket: 'videos',
+ playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
+ webtorrentBucket: objectStorage.getMockBucketName('videos'),
fixture
})
})
})
it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () {
- this.timeout(120000)
+ this.timeout(160000)
await servers[1].videos.upload({ attributes: { name: 'server2' } })
await servers[2].videos.upload({ attributes: { name: 'server3' } })
describe('Object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
this.timeout(30000)
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
})
it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
this.timeout(120000)
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv })
+ await servers[0].run(objectStorage.getDefaultMockConfig(), { env: goodEnv })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
- expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
})
it('Should fail to upload to object storage with a wrong proxy config', async function () {
this.timeout(120000)
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv })
+ await servers[0].run(objectStorage.getDefaultMockConfig(), { env: badEnv })
const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
await waitJobs(servers, { skipDelayed: true })
const video = await servers[0].videos.get({ id: uuid })
- expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectNotStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
+ })
+
+ after(async function () {
+ await objectStorage.cleanupMock()
})
})
waitJobs
} from '@shared/server-commands'
-async function checkFilesInObjectStorage (video: VideoDetails) {
+async function checkFilesInObjectStorage (objectStorage: ObjectStorageCommand, video: VideoDetails) {
for (const file of video.files) {
- expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const hlsPlaylist = video.streamingPlaylists[0]
for (const file of hlsPlaylist.files) {
- expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
- expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(hlsPlaylist.playlistUrl, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
- expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(hlsPlaylist.segmentsSha256Url, objectStorage.getMockPlaylistBaseUrl())
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
-function runTests (objectStorage: boolean) {
+function runTests (enableObjectStorage: boolean) {
let servers: PeerTubeServer[] = []
let videoUUID: string
let publishedAt: string
let shouldBeDeleted: string[]
+ const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
- const config = objectStorage
- ? ObjectStorageCommand.getDefaultMockConfig()
+ const config = enableObjectStorage
+ ? objectStorage.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
await doubleFollow(servers[0], servers[1])
- if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
+ if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets()
const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
videoUUID = shortUUID
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
- if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+ if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
- if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+ if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
- if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+ if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
expect(videoDetails.files).to.have.lengthOf(5)
expect(videoDetails.streamingPlaylists).to.have.lengthOf(0)
- if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+ if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
}
})
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1)
- if (objectStorage) await checkFilesInObjectStorage(videoDetails)
+ if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails)
shouldBeDeleted = [
videoDetails.streamingPlaylists[0].files[0].fileUrl,
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5)
- if (objectStorage) {
- await checkFilesInObjectStorage(videoDetails)
+ if (enableObjectStorage) {
+ await checkFilesInObjectStorage(objectStorage, videoDetails)
const hlsPlaylist = videoDetails.streamingPlaylists[0]
const resolutions = hlsPlaylist.files.map(f => f.resolution.id)
})
after(async function () {
+ if (objectStorage) await objectStorage.cleanupMock()
+
await cleanupTests(servers)
})
}
describe('With object storage enabled', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
this.timeout(120000)
- const configOverride = ObjectStorageCommand.getDefaultMockConfig()
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ const configOverride = objectStorage.getDefaultMockConfig()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
- runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ runTestSuite(true, objectStorage.getMockPlaylistBaseUrl())
+
+ after(async function () {
+ await objectStorage.cleanupMock()
+ })
})
after(async function () {
describe('With object storage enabled', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
this.timeout(120000)
- const configOverride = ObjectStorageCommand.getDefaultMockConfig()
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ const configOverride = objectStorage.getDefaultMockConfig()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
- runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ runTestSuite(true, objectStorage.getMockPlaylistBaseUrl())
+
+ after(async function () {
+ await objectStorage.cleanupMock()
+ })
})
after(async function () {
describe('Object storage studio edition', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].run(objectStorage.getDefaultMockConfig())
await servers[0].config.enableMinimumTranscoding()
})
}
for (const webtorrentFile of video.files) {
- expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(webtorrentFile.fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
for (const hlsFile of video.streamingPlaylists[0].files) {
- expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
await checkVideoDuration(server, videoUUID, 9)
}
})
+
+ after(async function () {
+ await objectStorage.cleanupMock()
+ })
})
after(async function () {
if (size) expect(video.size).to.equal(size)
}
-async function checkFiles (video: VideoDetails, objectStorage: boolean) {
+async function checkFiles (video: VideoDetails, objectStorage: ObjectStorageCommand) {
for (const file of video.files) {
- if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ if (objectStorage) expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
-function runTests (objectStorage: boolean) {
+function runTests (enableObjectStorage: boolean) {
let video1ShortId: string
let video2UUID: string
let servers: PeerTubeServer[] = []
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
this.timeout(90000)
- const config = objectStorage
- ? ObjectStorageCommand.getDefaultMockConfig()
+ const config = enableObjectStorage
+ ? objectStorage.getDefaultMockConfig()
: {}
// Run server 2 to have transcoding enabled
await doubleFollow(servers[0], servers[1])
- if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
+ if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets()
// Upload two videos for our needs
{
assertVideoProperties(originalVideo, 720, 'webm', 218910)
assertVideoProperties(transcodedVideo, 480, 'webm', 69217)
- await checkFiles(videoDetails, objectStorage)
+ await checkFiles(videoDetails, enableObjectStorage && objectStorage)
}
})
assertVideoProperties(transcodedVideo320, 360, 'mp4')
assertVideoProperties(transcodedVideo240, 240, 'mp4')
- await checkFiles(videoDetails, objectStorage)
+ await checkFiles(videoDetails, enableObjectStorage && objectStorage)
}
})
assertVideoProperties(video720, 720, 'webm', 942961)
assertVideoProperties(video480, 480, 'webm', 69217)
- await checkFiles(videoDetails, objectStorage)
+ await checkFiles(videoDetails, enableObjectStorage && objectStorage)
}
})
})
after(async function () {
+ await objectStorage.cleanupMock()
+
await cleanupTests(servers)
})
}
} from '@shared/server-commands'
import { checkDirectoryIsEmpty, expectStartWith } from '../shared'
-async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
+async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) {
for (const file of video.files) {
- const start = inObjectStorage
- ? ObjectStorageCommand.getMockWebTorrentBaseUrl()
+ const start = objectStorage
+ ? objectStorage.getMockWebVideosBaseUrl()
: origin.url
expectStartWith(file.fileUrl, start)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
- const start = inObjectStorage
- ? ObjectStorageCommand.getMockPlaylistBaseUrl()
+ const start = objectStorage
+ ? objectStorage.getMockPlaylistBaseUrl()
: origin.url
const hls = video.streamingPlaylists[0]
let servers: PeerTubeServer[] = []
const uuids: string[] = []
+ const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(360000)
await doubleFollow(servers[0], servers[1])
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].config.enableTranscoding()
await waitJobs(servers)
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].run(objectStorage.getDefaultMockConfig())
})
it('Should move only one file', async function () {
this.timeout(120000)
const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
- await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuids[1] })
- await checkFiles(servers[0], video, true)
+ await checkFiles(servers[0], video, objectStorage)
for (const id of [ uuids[0], uuids[2] ]) {
const video = await server.videos.get({ id })
- await checkFiles(servers[0], video, false)
+ await checkFiles(servers[0], video)
}
}
})
this.timeout(120000)
const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
- await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig())
await waitJobs(servers)
for (const server of servers) {
for (const id of [ uuids[0], uuids[2] ]) {
const video = await server.videos.get({ id })
- await checkFiles(servers[0], video, true)
+ await checkFiles(servers[0], video, objectStorage)
}
}
})
let sqlCommandServer1: SQLCommand
function runSuite (options: {
- objectStorage: boolean
- }) {
+ objectStorage?: ObjectStorageCommand
+ } = {}) {
const { objectStorage } = options
it('Should enable transcoding without additional resolutions', async function () {
for (const file of files) {
if (objectStorage) {
- expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
await servers[0].config.enableTranscoding(true, false, true)
})
- runSuite({ objectStorage: false })
+ runSuite()
})
describe('With lives on object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].run(objectStorage.getDefaultMockConfig())
// Wait for peertube runner socket reconnection
await wait(1500)
})
- runSuite({ objectStorage: true })
+ runSuite({ objectStorage })
+
+ after(async function () {
+ await objectStorage.cleanupMock()
+ })
})
describe('Check cleanup', function () {
let peertubeRunner: PeerTubeRunnerProcess
function runSuite (options: {
- objectStorage: boolean
- }) {
+ objectStorage?: ObjectStorageCommand
+ } = {}) {
const { objectStorage } = options
it('Should run a complex studio transcoding', async function () {
if (objectStorage) {
for (const webtorrentFile of video.files) {
- expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
+ expectStartWith(webtorrentFile.fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
for (const hlsFile of video.streamingPlaylists[0].files) {
- expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
+ expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
}
})
describe('With videos on local filesystem storage', function () {
- runSuite({ objectStorage: false })
+ runSuite()
})
describe('With videos on object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].run(objectStorage.getDefaultMockConfig())
// Wait for peertube runner socket reconnection
await wait(1500)
})
- runSuite({ objectStorage: true })
+ runSuite({ objectStorage })
+
+ after(async function () {
+ await objectStorage.cleanupMock()
+ })
})
describe('Check cleanup', function () {
function runSuite (options: {
webtorrentEnabled: boolean
hlsEnabled: boolean
- objectStorage: boolean
+ objectStorage?: ObjectStorageCommand
}) {
const { webtorrentEnabled, hlsEnabled, objectStorage } = options
const objectStorageBaseUrlWebTorrent = objectStorage
- ? ObjectStorageCommand.getMockWebTorrentBaseUrl()
+ ? objectStorage.getMockWebVideosBaseUrl()
: undefined
const objectStorageBaseUrlHLS = objectStorage
- ? ObjectStorageCommand.getMockPlaylistBaseUrl()
+ ? objectStorage.getMockPlaylistBaseUrl()
: undefined
it('Should upload a classic video mp4 and transcode it', async function () {
await servers[0].config.enableTranscoding(true, false, true)
})
- runSuite({ webtorrentEnabled: true, hlsEnabled: false, objectStorage: false })
+ runSuite({ webtorrentEnabled: true, hlsEnabled: false })
})
describe('HLS videos only enabled', function () {
await servers[0].config.enableTranscoding(false, true, true)
})
- runSuite({ webtorrentEnabled: false, hlsEnabled: true, objectStorage: false })
+ runSuite({ webtorrentEnabled: false, hlsEnabled: true })
})
describe('Web video & HLS enabled', function () {
await servers[0].config.enableTranscoding(true, true, true)
})
- runSuite({ webtorrentEnabled: true, hlsEnabled: true, objectStorage: false })
+ runSuite({ webtorrentEnabled: true, hlsEnabled: true })
})
})
describe('With videos on object storage', function () {
if (areMockObjectStorageTestsDisabled()) return
+ const objectStorage = new ObjectStorageCommand()
+
before(async function () {
- await ObjectStorageCommand.prepareDefaultMockBuckets()
+ await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
- await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
+ await servers[0].run(objectStorage.getDefaultMockConfig())
// Wait for peertube runner socket reconnection
await wait(1500)
await servers[0].config.enableTranscoding(true, false, true)
})
- runSuite({ webtorrentEnabled: true, hlsEnabled: false, objectStorage: true })
+ runSuite({ webtorrentEnabled: true, hlsEnabled: false, objectStorage })
})
describe('HLS videos only enabled', function () {
await servers[0].config.enableTranscoding(false, true, true)
})
- runSuite({ webtorrentEnabled: false, hlsEnabled: true, objectStorage: true })
+ runSuite({ webtorrentEnabled: false, hlsEnabled: true, objectStorage })
})
describe('Web video & HLS enabled', function () {
await servers[0].config.enableTranscoding(true, true, true)
})
- runSuite({ webtorrentEnabled: true, hlsEnabled: true, objectStorage: true })
+ runSuite({ webtorrentEnabled: true, hlsEnabled: true, objectStorage })
+ })
+
+ after(async function () {
+ await objectStorage.cleanupMock()
})
})
resolutions: number[]
transcoded: boolean
- objectStorage: boolean
+ objectStorage?: ObjectStorageCommand
objectStorageBaseUrl?: string
}) {
const {
resolutions,
transcoded,
objectStorage,
- objectStorageBaseUrl = ObjectStorageCommand.getMockPlaylistBaseUrl()
+ objectStorageBaseUrl = objectStorage?.getMockPlaylistBaseUrl()
} = options
for (const server of servers) {
playlistUrl: hlsPlaylist.playlistUrl,
resolutions,
transcoded,
- withRetry: objectStorage
+ withRetry: !!objectStorage
})
if (objectStorage) {
const subPlaylist = await originServer.streamingPlaylists.get({
url: `${baseUrl}/${video.uuid}/${i}.m3u8`,
- withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
+ withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3
})
expect(subPlaylist).to.contain(segmentName)
videoUUID: video.uuid,
segmentName,
hlsPlaylist,
- withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3
+ withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3
})
if (originServer.internalServerNumber === server.internalServerNumber) {
-
+import { randomInt } from 'crypto'
import { HttpStatusCode } from '@shared/models'
import { makePostBodyRequest } from '../requests'
-import { AbstractCommand } from '../shared'
-
-export class ObjectStorageCommand extends AbstractCommand {
- static readonly DEFAULT_PLAYLIST_MOCK_BUCKET = 'streaming-playlists'
- static readonly DEFAULT_WEBTORRENT_MOCK_BUCKET = 'videos'
+export class ObjectStorageCommand {
static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test'
- // ---------------------------------------------------------------------------
-
- static getDefaultMockConfig () {
- return {
- object_storage: {
- enabled: true,
- endpoint: 'http://' + this.getMockEndpointHost(),
- region: this.getMockRegion(),
-
- credentials: this.getMockCredentialsConfig(),
+ private readonly bucketsCreated: string[] = []
+ private readonly seed: number
- streaming_playlists: {
- bucket_name: this.DEFAULT_PLAYLIST_MOCK_BUCKET
- },
+ // ---------------------------------------------------------------------------
- videos: {
- bucket_name: this.DEFAULT_WEBTORRENT_MOCK_BUCKET
- }
- }
- }
+ constructor () {
+ this.seed = randomInt(0, 10000)
}
static getMockCredentialsConfig () {
return 'us-east-1'
}
- static getMockWebTorrentBaseUrl () {
- return `http://${this.DEFAULT_WEBTORRENT_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
+ getDefaultMockConfig () {
+ return {
+ object_storage: {
+ enabled: true,
+ endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
+ region: ObjectStorageCommand.getMockRegion(),
+
+ credentials: ObjectStorageCommand.getMockCredentialsConfig(),
+
+ streaming_playlists: {
+ bucket_name: this.getMockStreamingPlaylistsBucketName()
+ },
+
+ videos: {
+ bucket_name: this.getMockWebVideosBucketName()
+ }
+ }
+ }
}
- static getMockPlaylistBaseUrl () {
- return `http://${this.DEFAULT_PLAYLIST_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
+ getMockWebVideosBaseUrl () {
+ return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
- static async prepareDefaultMockBuckets () {
- await this.createMockBucket(this.DEFAULT_PLAYLIST_MOCK_BUCKET)
- await this.createMockBucket(this.DEFAULT_WEBTORRENT_MOCK_BUCKET)
+ getMockPlaylistBaseUrl () {
+ return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
- static async createMockBucket (name: string) {
+ async prepareDefaultMockBuckets () {
+ await this.createMockBucket(this.getMockStreamingPlaylistsBucketName())
+ await this.createMockBucket(this.getMockWebVideosBucketName())
+ }
+
+ async createMockBucket (name: string) {
+ this.bucketsCreated.push(name)
+
+ await this.deleteMockBucket(name)
+
await makePostBodyRequest({
- url: this.getMockEndpointHost(),
- path: '/ui/' + name + '?delete',
+ url: ObjectStorageCommand.getMockEndpointHost(),
+ path: '/ui/' + name + '?create',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
await makePostBodyRequest({
- url: this.getMockEndpointHost(),
- path: '/ui/' + name + '?create',
+ url: ObjectStorageCommand.getMockEndpointHost(),
+ path: '/ui/' + name + '?make-public',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
+ }
+ async cleanupMock () {
+ for (const name of this.bucketsCreated) {
+ await this.deleteMockBucket(name)
+ }
+ }
+
+ getMockStreamingPlaylistsBucketName (name = 'streaming-playlists') {
+ return this.getMockBucketName(name)
+ }
+
+ getMockWebVideosBucketName (name = 'web-videos') {
+ return this.getMockBucketName(name)
+ }
+
+ getMockBucketName (name: string) {
+ return `${this.seed}-${name}`
+ }
+
+ private async deleteMockBucket (name: string) {
await makePostBodyRequest({
- url: this.getMockEndpointHost(),
- path: '/ui/' + name + '?make-public',
+ url: ObjectStorageCommand.getMockEndpointHost(),
+ path: '/ui/' + name + '?delete',
expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
})
}
import { FollowsCommand } from './follows-command'
import { JobsCommand } from './jobs-command'
import { MetricsCommand } from './metrics-command'
-import { ObjectStorageCommand } from './object-storage-command'
import { PluginsCommand } from './plugins-command'
import { RedundancyCommand } from './redundancy-command'
import { ServersCommand } from './servers-command'
servers?: ServersCommand
login?: LoginCommand
users?: UsersCommand
- objectStorage?: ObjectStorageCommand
videoStudio?: VideoStudioCommand
videos?: VideosCommand
videoStats?: VideoStatsCommand
this.login = new LoginCommand(this)
this.users = new UsersCommand(this)
this.videos = new VideosCommand(this)
- this.objectStorage = new ObjectStorageCommand(this)
this.videoStudio = new VideoStudioCommand(this)
this.videoStats = new VideoStatsCommand(this)
this.views = new ViewsCommand(this)
videoUUID: string
playlistNumber: number
segment: number
- objectStorage: boolean
+ objectStorage?: ObjectStorageCommand
objectStorageBaseUrl?: string
}) {
const {
playlistNumber,
segment,
videoUUID,
- objectStorageBaseUrl = ObjectStorageCommand.getMockPlaylistBaseUrl()
+ objectStorageBaseUrl
} = options
const segmentName = `${playlistNumber}-00000${segment}.ts`
const baseUrl = objectStorage
- ? join(objectStorageBaseUrl, 'hls')
+ ? join(objectStorageBaseUrl || objectStorage.getMockPlaylistBaseUrl(), 'hls')
: server.url + '/static/streaming-playlists/hls'
let error = true
const hlsPlaylist = video.streamingPlaylists[0]
// Check SHA generation
- const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: objectStorage })
+ const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: !!objectStorage })
if (!shaBody[segmentName]) {
throw new Error('Segment SHA does not exist')
}
videoUUID: string
playlistNumber: number
segment: number
- objectStorage?: boolean // default false
+ objectStorage?: ObjectStorageCommand
}) {
- const { playlistNumber, segment, videoUUID, objectStorage = false } = options
+ const { playlistNumber, segment, videoUUID, objectStorage } = options
const segmentName = `${playlistNumber}-00000${segment}.ts`
const baseUrl = objectStorage
- ? ObjectStorageCommand.getMockPlaylistBaseUrl()
+ ? objectStorage.getMockPlaylistBaseUrl()
: `${this.server.url}/static/streaming-playlists/hls`
const url = `${baseUrl}/${videoUUID}/${segmentName}`
getPlaylistFile (options: OverrideCommandOptions & {
videoUUID: string
playlistName: string
- objectStorage?: boolean // default false
+ objectStorage?: ObjectStorageCommand
}) {
- const { playlistName, videoUUID, objectStorage = false } = options
+ const { playlistName, videoUUID, objectStorage } = options
const baseUrl = objectStorage
- ? ObjectStorageCommand.getMockPlaylistBaseUrl()
+ ? objectStorage.getMockPlaylistBaseUrl()
: `${this.server.url}/static/streaming-playlists/hls`
const url = `${baseUrl}/${videoUUID}/${playlistName}`
projects / github / Chocobozzz / PeerTube.git / commitdiff
