/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
-import 'mocha'
-import * as chai from 'chai'
+import bytes from 'bytes'
+import { expect } from 'chai'
+import { stat } from 'fs-extra'
import { merge } from 'lodash'
import {
- areObjectStorageTestsDisabled,
checkTmpIsEmpty,
+ checkWebTorrentWorks,
+ expectLogDoesNotContain,
+ expectStartWith,
+ generateHighBitrateVideo,
+ MockObjectStorageProxy,
+ SQLCommand
+} from '@server/tests/shared'
+import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
+import { sha1 } from '@shared/extra-utils'
+import { HttpStatusCode, VideoDetails } from '@shared/models'
+import {
cleanupTests,
createMultipleServers,
createSingleServer,
doubleFollow,
- expectStartWith,
killallServers,
makeRawRequest,
- MockObjectStorage,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
- waitJobs,
- webtorrentAdd
-} from '@shared/extra-utils'
-import { HttpStatusCode, VideoDetails } from '@shared/models'
-
-const expect = chai.expect
+ waitJobs
+} from '@shared/server-commands'
async function checkFiles (options: {
+ server: PeerTubeServer
+ originServer: PeerTubeServer
+ originSQLCommand: SQLCommand
+
video: VideoDetails
baseMockUrl?: string
webtorrentPrefix?: string
}) {
const {
+ server,
+ originServer,
+ originSQLCommand,
video,
playlistBucket,
webtorrentBucket,
for (const file of video.files) {
const baseUrl = baseMockUrl
? `${baseMockUrl}/${webtorrentBucket}/`
- : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
+ : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
const prefix = webtorrentPrefix || ''
const start = baseUrl + prefix
expectStartWith(file.fileUrl, start)
- const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
+ const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
const location = res.headers['location']
expectStartWith(location, start)
- await makeRawRequest(location, HttpStatusCode.OK_200)
+ await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
const baseUrl = baseMockUrl
? `${baseMockUrl}/${playlistBucket}/`
- : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
+ : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
const prefix = playlistPrefix || ''
const start = baseUrl + prefix
expectStartWith(hls.playlistUrl, start)
expectStartWith(hls.segmentsSha256Url, start)
- await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
+ await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
- const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
+ const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
expect(JSON.stringify(resSha.body)).to.not.throw
+ let i = 0
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
- const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
+ const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
const location = res.headers['location']
expectStartWith(location, start)
- await makeRawRequest(location, HttpStatusCode.OK_200)
+ await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
+
+ if (originServer.internalServerNumber === server.internalServerNumber) {
+ const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
+ const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id)
+
+ expect(dbInfohashes).to.include(infohash)
+ }
+
+ i++
}
}
for (const file of allFiles) {
- const torrent = await webtorrentAdd(file.magnetUri, true)
+ await checkWebTorrentWorks(file.magnetUri)
- expect(torrent.files).to.be.an('array')
- expect(torrent.files.length).to.equal(1)
- expect(torrent.files[0].path).to.exist.and.to.not.equal('')
-
- const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
+ const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.length.above(100)
}
}
function runTestSuite (options: {
+ fixture?: string
+
+ maxUploadPart?: string
+
playlistBucket: string
playlistPrefix?: string
webtorrentPrefix?: string
useMockBaseUrl?: boolean
-
- maxUploadPart?: string
}) {
- const mockObjectStorage = new MockObjectStorage()
+ const mockObjectStorageProxy = new MockObjectStorageProxy()
+ const { fixture } = options
let baseMockUrl: string
let servers: PeerTubeServer[]
+ let sqlCommands: SQLCommand[] = []
let keptUrls: string[] = []
let deletedUrls: string[] = []
before(async function () {
- this.timeout(120000)
+ this.timeout(240000)
- const port = await mockObjectStorage.initialize()
- baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
+ const port = await mockObjectStorageProxy.initialize()
+ baseMockUrl = options.useMockBaseUrl
+ ? `http://127.0.0.1:${port}`
+ : undefined
- await ObjectStorageCommand.createBucket(options.playlistBucket)
- await ObjectStorageCommand.createBucket(options.webtorrentBucket)
+ await ObjectStorageCommand.createMockBucket(options.playlistBucket)
+ await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
const config = {
object_storage: {
enabled: true,
- endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
- region: ObjectStorageCommand.getRegion(),
+ endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
+ region: ObjectStorageCommand.getMockRegion(),
- credentials: ObjectStorageCommand.getCredentialsConfig(),
+ credentials: ObjectStorageCommand.getMockCredentialsConfig(),
- max_upload_part: options.maxUploadPart || '2MB',
+ max_upload_part: options.maxUploadPart || '5MB',
streaming_playlists: {
bucket_name: options.playlistBucket,
const files = await server.videos.listFiles({ id: uuid })
keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
}
+
+ sqlCommands = servers.map(s => new SQLCommand(s))
})
it('Should upload a video and move it to the object storage without transcoding', async function () {
- this.timeout(20000)
+ this.timeout(40000)
- const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
uuidsToDelete.push(uuid)
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
- const files = await checkFiles({ ...options, video, baseMockUrl })
+ const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
deletedUrls = deletedUrls.concat(files)
}
})
it('Should upload a video and move it to the object storage with transcoding', async function () {
- this.timeout(40000)
+ this.timeout(120000)
- const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
+ const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
uuidsToDelete.push(uuid)
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
- const files = await checkFiles({ ...options, video, baseMockUrl })
+ const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
deletedUrls = deletedUrls.concat(files)
}
})
+ it('Should fetch correctly all the files', async function () {
+ for (const url of deletedUrls.concat(keptUrls)) {
+ await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
+ }
+ })
+
it('Should correctly delete the files', async function () {
await servers[0].videos.remove({ id: uuidsToDelete[0] })
await servers[1].videos.remove({ id: uuidsToDelete[1] })
await waitJobs(servers)
for (const url of deletedUrls) {
- await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
+ await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
})
it('Should have kept other files', async function () {
for (const url of keptUrls) {
- await makeRawRequest(url, HttpStatusCode.OK_200)
+ await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
})
}
})
+ it('Should not have downloaded files from object storage', async function () {
+ for (const server of servers) {
+ await expectLogDoesNotContain(server, 'from object storage')
+ }
+ })
+
after(async function () {
- mockObjectStorage.terminate()
+ await mockObjectStorageProxy.terminate()
+
+ for (const sqlCommand of sqlCommands) {
+ await sqlCommand.cleanup()
+ }
await cleanupTests(servers)
})
}
describe('Object storage for videos', function () {
- if (areObjectStorageTestsDisabled()) return
+ if (areMockObjectStorageTestsDisabled()) return
describe('Test config', function () {
let server: PeerTubeServer
const baseConfig = {
object_storage: {
enabled: true,
- endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
- region: ObjectStorageCommand.getRegion(),
+ endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
+ region: ObjectStorageCommand.getMockRegion(),
- credentials: ObjectStorageCommand.getCredentialsConfig(),
+ credentials: ObjectStorageCommand.getMockCredentialsConfig(),
streaming_playlists: {
- bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
+ bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
},
videos: {
- bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
+ bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
}
}
}
it('Should fail with bad credentials', async function () {
this.timeout(60000)
- await ObjectStorageCommand.prepareDefaultBuckets()
+ await ObjectStorageCommand.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
const { uuid } = await server.videos.quickUpload({ name: 'video' })
- await waitJobs([ server ], true)
+ await waitJobs([ server ], { skipDelayed: true })
const video = await server.videos.get({ id: uuid })
expectStartWith(video.files[0].fileUrl, server.url)
it('Should succeed with credentials from env', async function () {
this.timeout(60000)
- await ObjectStorageCommand.prepareDefaultBuckets()
+ await ObjectStorageCommand.prepareDefaultMockBuckets()
const config = merge({}, baseConfig, {
object_storage: {
}
})
- const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
+ const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
server = await createSingleServer(1, config, {
env: {
const { uuid } = await server.videos.quickUpload({ name: 'video' })
- await waitJobs([ server ], true)
+ await waitJobs([ server ], { skipDelayed: true })
const video = await server.videos.get({ id: uuid })
- expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
+ expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
})
after(async function () {
- await killallServers([ server ])
+ await cleanupTests([ server ])
})
})
playlistBucket: 'mybucket',
webtorrentBucket: 'mybucket',
- playlistPrefix: 'streaming-playlists_',
- webtorrentPrefix: 'webtorrent_',
+ playlistPrefix: 'streaming-playlists/',
+ webtorrentPrefix: 'webtorrent/',
useMockBaseUrl: true
})
})
- describe('Test object storage with small upload part', function () {
+ describe('Test object storage with file bigger than upload part', function () {
+ let fixture: string
+ const maxUploadPart = '5MB'
+
+ before(async function () {
+ this.timeout(120000)
+
+ fixture = await generateHighBitrateVideo()
+
+ const { size } = await stat(fixture)
+
+ if (bytes.parse(maxUploadPart) > size) {
+ throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
+ }
+ })
+
runTestSuite({
+ maxUploadPart,
playlistBucket: 'streaming-playlists',
webtorrentBucket: 'videos',
-
- maxUploadPart: '5KB'
+ fixture
})
})
})
projects / github / Chocobozzz / PeerTube.git / blobdiff