From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- .../tests/api/runners/runner-live-transcoding.ts | 330 --------------------- 1 file changed, 330 deletions(-) delete mode 100644 server/tests/api/runners/runner-live-transcoding.ts (limited to 'server/tests/api/runners/runner-live-transcoding.ts') diff --git a/server/tests/api/runners/runner-live-transcoding.ts b/server/tests/api/runners/runner-live-transcoding.ts deleted file mode 100644 index b11d54039..000000000 --- a/server/tests/api/runners/runner-live-transcoding.ts +++ /dev/null @@ -1,330 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { readFile } from 'fs-extra' -import { buildAbsoluteFixturePath, wait } from '@shared/core-utils' -import { - HttpStatusCode, - LiveRTMPHLSTranscodingUpdatePayload, - LiveVideo, - LiveVideoError, - RunnerJob, - RunnerJobLiveRTMPHLSTranscodingPayload, - Video, - VideoPrivacy, - VideoState -} from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeRawRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - testFfmpegStreamError, - waitJobs -} from '@shared/server-commands' - -describe('Test runner live transcoding', function () { - let server: PeerTubeServer - let runnerToken: string - let baseUrl: string - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableRemoteTranscoding() - await server.config.enableTranscoding() - runnerToken = await server.runners.autoRegisterRunner() - - baseUrl = server.url + '/static/streaming-playlists/hls' - }) - - describe('Without transcoding enabled', function () { - - before(async function () { - await server.config.enableLive({ - allowReplay: false, - resolutions: 'min', - transcoding: false - }) - }) - - it('Should not have available jobs', async function () { - this.timeout(120000) - - const { live, video } = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await server.live.waitUntilPublished({ videoId: video.id }) - - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.requestLive({ runnerToken }) - expect(availableJobs).to.have.lengthOf(0) - - await stopFfmpeg(ffmpegCommand) - }) - }) - - describe('With transcoding enabled on classic live', function () { - let live: LiveVideo - let video: Video - let ffmpegCommand: FfmpegCommand - let jobUUID: string - let acceptedJob: RunnerJob & { jobToken: string } - - async function testPlaylistFile (fixture: string, expected: string) { - const text = await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${fixture}` }) - expect(await readFile(buildAbsoluteFixturePath(expected), 'utf-8')).to.equal(text) - - } - - async function testTSFile (fixture: string, expected: string) { - const { body } = await makeRawRequest({ url: `${baseUrl}/${video.uuid}/${fixture}`, expectedStatus: HttpStatusCode.OK_200 }) - expect(await readFile(buildAbsoluteFixturePath(expected))).to.deep.equal(body) - } - - before(async function () { - await server.config.enableLive({ - allowReplay: true, - resolutions: 'max', - transcoding: true - }) - }) - - it('Should publish a a live and have available jobs', async function () { - this.timeout(120000) - - const data = await server.live.quickCreate({ permanentLive: false, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - live = data.live - video = data.video - - ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await waitJobs([ server ]) - - const job = await server.runnerJobs.requestLiveJob(runnerToken) - jobUUID = job.uuid - - expect(job.type).to.equal('live-rtmp-hls-transcoding') - expect(job.payload.input.rtmpUrl).to.exist - - expect(job.payload.output.toTranscode).to.have.lengthOf(5) - - for (const { resolution, fps } of job.payload.output.toTranscode) { - expect([ 720, 480, 360, 240, 144 ]).to.contain(resolution) - - expect(fps).to.be.above(25) - expect(fps).to.be.below(70) - } - }) - - it('Should update the live with a new chunk', async function () { - this.timeout(120000) - - const { job } = await server.runnerJobs.accept({ jobUUID, runnerToken }) - acceptedJob = job - - { - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - masterPlaylistFile: 'live/master.m3u8', - resolutionPlaylistFile: 'live/0.m3u8', - resolutionPlaylistFilename: '0.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/0-000067.ts', - videoChunkFilename: '0-000067.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload, progress: 50 }) - - const updatedJob = await server.runnerJobs.getJob({ uuid: job.uuid }) - expect(updatedJob.progress).to.equal(50) - } - - { - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - resolutionPlaylistFile: 'live/1.m3u8', - resolutionPlaylistFilename: '1.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/1-000068.ts', - videoChunkFilename: '1-000068.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload }) - } - - await wait(1000) - - await testPlaylistFile('master.m3u8', 'live/master.m3u8') - await testPlaylistFile('0.m3u8', 'live/0.m3u8') - await testPlaylistFile('1.m3u8', 'live/1.m3u8') - - await testTSFile('0-000067.ts', 'live/0-000067.ts') - await testTSFile('1-000068.ts', 'live/1-000068.ts') - }) - - it('Should replace existing m3u8 on update', async function () { - this.timeout(120000) - - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - masterPlaylistFile: 'live/1.m3u8', - resolutionPlaylistFilename: '0.m3u8', - resolutionPlaylistFile: 'live/1.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/1-000069.ts', - videoChunkFilename: '1-000068.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) - await wait(1000) - - await testPlaylistFile('master.m3u8', 'live/1.m3u8') - await testPlaylistFile('0.m3u8', 'live/1.m3u8') - await testTSFile('1-000068.ts', 'live/1-000069.ts') - }) - - it('Should update the live with removed chunks', async function () { - this.timeout(120000) - - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - resolutionPlaylistFile: 'live/0.m3u8', - resolutionPlaylistFilename: '0.m3u8', - type: 'remove-chunk', - videoChunkFilename: '1-000068.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) - - await wait(1000) - - await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/master.m3u8` }) - await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/0.m3u8` }) - await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/1.m3u8` }) - await makeRawRequest({ url: `${baseUrl}/${video.uuid}/0-000067.ts`, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: `${baseUrl}/${video.uuid}/1-000068.ts`, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should complete the live and save the replay', async function () { - this.timeout(120000) - - for (const segment of [ '0-000069.ts', '0-000070.ts' ]) { - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - masterPlaylistFile: 'live/master.m3u8', - resolutionPlaylistFilename: '0.m3u8', - resolutionPlaylistFile: 'live/0.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/' + segment, - videoChunkFilename: segment - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) - - await wait(1000) - } - - await waitJobs([ server ]) - - { - const { state } = await server.videos.get({ id: video.uuid }) - expect(state.id).to.equal(VideoState.PUBLISHED) - } - - await stopFfmpeg(ffmpegCommand) - - await server.runnerJobs.success({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload: {} }) - - await wait(1500) - await waitJobs([ server ]) - - { - const { state } = await server.videos.get({ id: video.uuid }) - expect(state.id).to.equal(VideoState.LIVE_ENDED) - - const session = await server.live.findLatestSession({ videoId: video.uuid }) - expect(session.error).to.be.null - } - }) - }) - - describe('With transcoding enabled on cancelled/aborted/errored live', function () { - let live: LiveVideo - let video: Video - let ffmpegCommand: FfmpegCommand - - async function prepare () { - ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await server.runnerJobs.requestLiveJob(runnerToken) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'live-rtmp-hls-transcoding' }) - - return job - } - - async function checkSessionError (error: LiveVideoError) { - await wait(1500) - await waitJobs([ server ]) - - const session = await server.live.findLatestSession({ videoId: video.uuid }) - expect(session.error).to.equal(error) - } - - before(async function () { - await server.config.enableLive({ - allowReplay: true, - resolutions: 'max', - transcoding: true - }) - - const data = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - live = data.live - video = data.video - }) - - it('Should abort a running live', async function () { - this.timeout(120000) - - const job = await prepare() - - await Promise.all([ - server.runnerJobs.abort({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, reason: 'abort' }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - // Abort is not supported - await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR) - }) - - it('Should cancel a running live', async function () { - this.timeout(120000) - - const job = await prepare() - - await Promise.all([ - server.runnerJobs.cancelByAdmin({ jobUUID: job.uuid }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - await checkSessionError(LiveVideoError.RUNNER_JOB_CANCEL) - }) - - it('Should error a running live', async function () { - this.timeout(120000) - - const job = await prepare() - - await Promise.all([ - server.runnerJobs.error({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, message: 'error' }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) -- cgit v1.2.3