From ca5c612bfdd225433bcc6ace01c8024df3f674ba Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 24 Nov 2020 15:22:56 +0100 Subject: [PATCH] Add live transcoding bit rate tests --- server/initializers/checker-before-init.ts | 2 -- server/lib/video-transcoding-profiles.ts | 3 ++- server/tests/api/live/live.ts | 22 ++++++++++++++++-- server/tests/api/videos/audio-only.ts | 6 ++--- server/tests/api/videos/video-transcoder.ts | 25 ++++++++++++--------- server/tests/cli/optimize-old-videos.ts | 4 ++-- server/tests/cli/prune-storage.ts | 14 ++++++------ shared/extra-utils/miscs/miscs.ts | 8 +++---- shared/extra-utils/server/plugins.ts | 12 +++++----- shared/extra-utils/server/servers.ts | 6 ++--- shared/extra-utils/videos/live.ts | 10 ++++----- shared/extra-utils/videos/videos.ts | 15 ++++--------- 12 files changed, 70 insertions(+), 57 deletions(-) diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 4ffd6fad9..b04f3e538 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -82,8 +82,6 @@ function checkMissedConfig () { async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { if (CONFIG.TRANSCODING.ENABLED === false) return undefined - checkFFmpegEncoders() - const Ffmpeg = require('fluent-ffmpeg') const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) const codecs = await getAvailableCodecsPromise() diff --git a/server/lib/video-transcoding-profiles.ts b/server/lib/video-transcoding-profiles.ts index 12e22a19d..91a5c65f2 100644 --- a/server/lib/video-transcoding-profiles.ts +++ b/server/lib/video-transcoding-profiles.ts @@ -38,7 +38,8 @@ const defaultX264LiveOptionsBuilder: EncoderOptionsBuilder = async ({ resolution return { outputOptions: [ `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}`, - `-maxrate ${targetBitrate}`, `-bufsize ${targetBitrate * 2}` + `-maxrate ${targetBitrate}`, + `-bufsize ${targetBitrate * 2}` ] } } diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts index ac91b6b08..e685be08e 100644 --- a/server/tests/api/live/live.ts +++ b/server/tests/api/live/live.ts @@ -3,10 +3,13 @@ import 'mocha' import * as chai from 'chai' import { FfmpegCommand } from 'fluent-ffmpeg' +import { join } from 'path' +import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils' import { getLiveNotificationSocket } from '@shared/extra-utils/socket/socket-io' import { LiveVideo, LiveVideoCreate, Video, VideoDetails, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models' import { addVideoToBlacklist, + buildServerDirectory, checkLiveCleanup, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, @@ -396,10 +399,11 @@ describe('Test live', function () { this.timeout(60000) const resolutions = [ 240, 360, 720 ] + await updateConf(resolutions) liveVideoId = await createLiveWrapper(true) - const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId) + const command = await sendRTMPStreamInVideo(servers[0].url, servers[0].accessToken, liveVideoId, 'video_short2.webm') await waitUntilLiveStarts(servers[0].url, servers[0].accessToken, liveVideoId) await waitJobs(servers) @@ -409,6 +413,12 @@ describe('Test live', function () { await waitJobs(servers) + const bitrateLimits = { + 720: 2800 * 1000, + 360: 780 * 1000, + 240: 320 * 1000 + } + for (const server of servers) { const resVideo = await getVideo(server.url, liveVideoId) const video: VideoDetails = resVideo.body @@ -424,9 +434,17 @@ describe('Test live', function () { const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) expect(file).to.exist - expect(file.fps).to.be.oneOf([ 24, 25 ]) + expect(file.fps).to.be.approximately(30, 5) expect(file.size).to.be.greaterThan(1) + const filename = `${video.uuid}-${resolution}-fragmented.mp4` + const segmentPath = buildServerDirectory(servers[0], join('streaming-playlists', 'hls', video.uuid, filename)) + + const probe = await ffprobePromise(segmentPath) + const videoStream = await getVideoStreamFromFile(segmentPath, probe) + console.log(videoStream) + expect(probe.format.bit_rate).to.be.below(bitrateLimits[videoStream.height]) + await makeRawRequest(file.torrentUrl, 200) await makeRawRequest(file.fileUrl, 200) } diff --git a/server/tests/api/videos/audio-only.ts b/server/tests/api/videos/audio-only.ts index 053b29ca1..294a00112 100644 --- a/server/tests/api/videos/audio-only.ts +++ b/server/tests/api/videos/audio-only.ts @@ -5,11 +5,11 @@ import * as chai from 'chai' import { join } from 'path' import { getAudioStream, getVideoStreamSize } from '@server/helpers/ffprobe-utils' import { + buildServerDirectory, cleanupTests, doubleFollow, flushAndRunMultipleServers, getVideo, - root, ServerInfo, setAccessTokensToServers, uploadVideo, @@ -80,8 +80,8 @@ describe('Test audio only video transcoding', function () { it('0p transcoded video should not have video', async function () { const paths = [ - join(root(), 'test' + servers[0].internalServerNumber, 'videos', videoUUID + '-0.mp4'), - join(root(), 'test' + servers[0].internalServerNumber, 'streaming-playlists', 'hls', videoUUID, videoUUID + '-0-fragmented.mp4') + buildServerDirectory(servers[0], join('videos', videoUUID + '-0.mp4')), + buildServerDirectory(servers[0], join('streaming-playlists', 'hls', videoUUID, videoUUID + '-0-fragmented.mp4')) ] for (const path of paths) { diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 3e336e786..164843d32 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -5,10 +5,10 @@ import * as chai from 'chai' import { FfprobeData } from 'fluent-ffmpeg' import { omit } from 'lodash' import { join } from 'path' - import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants' import { buildAbsoluteFixturePath, + buildServerDirectory, cleanupTests, doubleFollow, flushAndRunMultipleServers, @@ -20,7 +20,6 @@ import { getVideoFileMetadataUrl, getVideosList, makeGetRequest, - root, ServerInfo, setAccessTokensToServers, updateCustomSubConfig, @@ -136,7 +135,7 @@ describe('Test video transcoding', function () { expect(videoDetails.files).to.have.lengthOf(4) - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-240.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-240.mp4')) const probe = await getAudioStream(path) if (probe.audioStream) { @@ -167,7 +166,7 @@ describe('Test video transcoding', function () { const videoDetails: VideoDetails = res2.body expect(videoDetails.files).to.have.lengthOf(4) - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-240.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-240.mp4')) const probe = await getAudioStream(path) expect(probe).to.not.have.property('audioStream') } @@ -192,10 +191,13 @@ describe('Test video transcoding', function () { const videoDetails: VideoDetails = res2.body expect(videoDetails.files).to.have.lengthOf(4) + const fixturePath = buildAbsoluteFixturePath(videoAttributes.fixture) const fixtureVideoProbe = await getAudioStream(fixturePath) - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-240.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-240.mp4')) + const videoProbe = await getAudioStream(path) + if (videoProbe.audioStream && fixtureVideoProbe.audioStream) { const toOmit = [ 'max_bit_rate', 'duration', 'duration_ts', 'nb_frames', 'start_time', 'start_pts' ] expect(omit(videoProbe.audioStream, toOmit)).to.be.deep.equal(omit(fixtureVideoProbe.audioStream, toOmit)) @@ -231,13 +233,13 @@ describe('Test video transcoding', function () { expect(videoDetails.files[3].fps).to.be.below(31) for (const resolution of [ '240', '360', '480' ]) { - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-' + resolution + '.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-' + resolution + '.mp4')) const fps = await getVideoFileFPS(path) expect(fps).to.be.below(31) } - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-720.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-720.mp4')) const fps = await getVideoFileFPS(path) expect(fps).to.be.above(58).and.below(62) @@ -325,7 +327,8 @@ describe('Test video transcoding', function () { const video = res.body.data.find(v => v.name === videoAttributes.name) for (const resolution of [ '240', '360', '480', '720', '1080' ]) { - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-' + resolution + '.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-' + resolution + '.mp4')) + const bitrate = await getVideoFileBitrate(path) const fps = await getVideoFileFPS(path) const resolution2 = await getVideoFileResolution(path) @@ -458,13 +461,13 @@ describe('Test video transcoding', function () { const video = res.body.data.find(v => v.name === videoAttributes.name) { - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-240.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-240.mp4')) const fps = await getVideoFileFPS(path) expect(fps).to.be.equal(25) } { - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', video.uuid + '-720.mp4') + const path = buildServerDirectory(servers[1], join('videos', video.uuid + '-720.mp4')) const fps = await getVideoFileFPS(path) expect(fps).to.be.equal(59) } @@ -513,7 +516,7 @@ describe('Test video transcoding', function () { await waitJobs(servers) { - const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', videoUUID + '-240.mp4') + const path = buildServerDirectory(servers[1], join('videos', videoUUID + '-240.mp4')) const metadata = await getMetadataFromFile(path) // expected format properties diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts index 420fb8049..91a1c9cc4 100644 --- a/server/tests/cli/optimize-old-videos.ts +++ b/server/tests/cli/optimize-old-videos.ts @@ -4,6 +4,7 @@ import 'mocha' import * as chai from 'chai' import { join } from 'path' import { + buildServerDirectory, cleanupTests, doubleFollow, execCLI, @@ -12,7 +13,6 @@ import { getEnvCli, getVideo, getVideosList, - root, ServerInfo, setAccessTokensToServers, uploadVideo, @@ -100,7 +100,7 @@ describe('Test optimize old videos', function () { expect(file.size).to.be.below(8000000) - const path = join(root(), 'test' + servers[0].internalServerNumber, 'videos', video.uuid + '-' + file.resolution.id + '.mp4') + const path = buildServerDirectory(servers[0], join('videos', video.uuid + '-' + file.resolution.id + '.mp4')) const bitrate = await getVideoFileBitrate(path) const fps = await getVideoFileFPS(path) const resolution = await getVideoFileResolution(path) diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 6cda80070..052a5f91e 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts @@ -28,13 +28,13 @@ import { join } from 'path' const expect = chai.expect async function countFiles (internalServerNumber: number, directory: string) { - const files = await readdir(buildServerDirectory(internalServerNumber, directory)) + const files = await readdir(buildServerDirectory({ internalServerNumber }, directory)) return files.length } async function assertNotExists (internalServerNumber: number, directory: string, substring: string) { - const files = await readdir(buildServerDirectory(internalServerNumber, directory)) + const files = await readdir(buildServerDirectory({ internalServerNumber }, directory)) for (const f of files) { expect(f).to.not.contain(substring) @@ -124,7 +124,7 @@ describe('Test prune storage scripts', function () { it('Should create some dirty files', async function () { for (let i = 0; i < 2; i++) { { - const base = buildServerDirectory(servers[0].internalServerNumber, 'videos') + const base = buildServerDirectory(servers[0], 'videos') const n1 = uuidv4() + '.mp4' const n2 = uuidv4() + '.webm' @@ -136,7 +136,7 @@ describe('Test prune storage scripts', function () { } { - const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents') + const base = buildServerDirectory(servers[0], 'torrents') const n1 = uuidv4() + '-240.torrent' const n2 = uuidv4() + '-480.torrent' @@ -148,7 +148,7 @@ describe('Test prune storage scripts', function () { } { - const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails') + const base = buildServerDirectory(servers[0], 'thumbnails') const n1 = uuidv4() + '.jpg' const n2 = uuidv4() + '.jpg' @@ -160,7 +160,7 @@ describe('Test prune storage scripts', function () { } { - const base = buildServerDirectory(servers[0].internalServerNumber, 'previews') + const base = buildServerDirectory(servers[0], 'previews') const n1 = uuidv4() + '.jpg' const n2 = uuidv4() + '.jpg' @@ -172,7 +172,7 @@ describe('Test prune storage scripts', function () { } { - const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars') + const base = buildServerDirectory(servers[0], 'avatars') const n1 = uuidv4() + '.png' const n2 = uuidv4() + '.jpg' diff --git a/shared/extra-utils/miscs/miscs.ts b/shared/extra-utils/miscs/miscs.ts index 3c8191ae8..429083fe9 100644 --- a/shared/extra-utils/miscs/miscs.ts +++ b/shared/extra-utils/miscs/miscs.ts @@ -1,11 +1,11 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import * as chai from 'chai' +import * as ffmpeg from 'fluent-ffmpeg' +import { ensureDir, pathExists, readFile, stat } from 'fs-extra' import { basename, dirname, isAbsolute, join, resolve } from 'path' import * as request from 'supertest' import * as WebTorrent from 'webtorrent' -import { ensureDir, pathExists, readFile, stat } from 'fs-extra' -import * as ffmpeg from 'fluent-ffmpeg' const expect = chai.expect let webtorrent: WebTorrent.Instance @@ -44,8 +44,8 @@ function root () { return root } -function buildServerDirectory (internalServerNumber: number, directory: string) { - return join(root(), 'test' + internalServerNumber, directory) +function buildServerDirectory (server: { internalServerNumber: number }, directory: string) { + return join(root(), 'test' + server.internalServerNumber, directory) } async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { diff --git a/shared/extra-utils/server/plugins.ts b/shared/extra-utils/server/plugins.ts index b6b5e3958..8f370f655 100644 --- a/shared/extra-utils/server/plugins.ts +++ b/shared/extra-utils/server/plugins.ts @@ -1,10 +1,10 @@ -import { makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests' -import { PluginType } from '../../models/plugins/plugin.type' -import { PeertubePluginIndexList } from '../../models/plugins/peertube-plugin-index-list.model' import { readJSON, writeJSON } from 'fs-extra' -import { ServerInfo } from './servers' -import { root } from '../miscs/miscs' import { join } from 'path' +import { PeertubePluginIndexList } from '../../models/plugins/peertube-plugin-index-list.model' +import { PluginType } from '../../models/plugins/plugin.type' +import { buildServerDirectory, root } from '../miscs/miscs' +import { makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests' +import { ServerInfo } from './servers' function listPlugins (parameters: { url: string @@ -216,7 +216,7 @@ function getPluginsCSS (url: string) { } function getPackageJSONPath (server: ServerInfo, npmName: string) { - return join(root(), 'test' + server.internalServerNumber, 'plugins', 'node_modules', npmName, 'package.json') + return buildServerDirectory(server, join('plugins', 'node_modules', npmName, 'package.json')) } function updatePluginPackageJSON (server: ServerInfo, npmName: string, json: any) { diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index 75e79cc41..1126a7d93 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts @@ -6,7 +6,7 @@ import { copy, pathExists, readdir, readFile, remove } from 'fs-extra' import { join } from 'path' import { randomInt } from '../../core-utils/miscs/miscs' import { VideoChannel } from '../../models/videos' -import { getFileSize, root, wait } from '../miscs/miscs' +import { buildServerDirectory, getFileSize, root, wait } from '../miscs/miscs' interface ServerInfo { app: ChildProcess @@ -309,7 +309,7 @@ function cleanupTests (servers: ServerInfo[]) { } async function waitUntilLog (server: ServerInfo, str: string, count = 1, strictCount = true) { - const logfile = join(root(), 'test' + server.internalServerNumber, 'logs/peertube.log') + const logfile = buildServerDirectory(server, 'logs/peertube.log') while (true) { const buf = await readFile(logfile) @@ -323,7 +323,7 @@ async function waitUntilLog (server: ServerInfo, str: string, count = 1, strictC } async function getServerFileSize (server: ServerInfo, subPath: string) { - const path = join(root(), 'test' + server.internalServerNumber, subPath) + const path = buildServerDirectory(server, subPath) return getFileSize(path) } diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts index 4bfcc583e..85c83c5bb 100644 --- a/shared/extra-utils/videos/live.ts +++ b/shared/extra-utils/videos/live.ts @@ -53,15 +53,15 @@ function createLive (url: string, token: string, fields: LiveVideoCreate, status }) } -async function sendRTMPStreamInVideo (url: string, token: string, videoId: number | string) { +async function sendRTMPStreamInVideo (url: string, token: string, videoId: number | string, fixtureName?: string) { const res = await getLive(url, token, videoId) const videoLive = res.body as LiveVideo - return sendRTMPStream(videoLive.rtmpUrl, videoLive.streamKey) + return sendRTMPStream(videoLive.rtmpUrl, videoLive.streamKey, fixtureName) } -function sendRTMPStream (rtmpBaseUrl: string, streamKey: string) { - const fixture = buildAbsoluteFixturePath('video_short.mp4') +function sendRTMPStream (rtmpBaseUrl: string, streamKey: string, fixtureName = 'video_short.mp4') { + const fixture = buildAbsoluteFixturePath(fixtureName) const command = ffmpeg(fixture) command.inputOption('-stream_loop -1') @@ -140,7 +140,7 @@ async function waitUntilLiveStarts (url: string, token: string, videoId: number } async function checkLiveCleanup (server: ServerInfo, videoUUID: string, resolutions: number[] = []) { - const basePath = buildServerDirectory(server.internalServerNumber, 'streaming-playlists') + const basePath = buildServerDirectory(server, 'streaming-playlists') const hlsPath = join(basePath, 'hls', videoUUID) if (resolutions.length === 0) { diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 29a646541..2b8c55acb 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -9,15 +9,7 @@ import { v4 as uuidv4 } from 'uuid' import validator from 'validator' import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' import { VideoDetails, VideoPrivacy } from '../../models/videos' -import { - buildAbsoluteFixturePath, - buildServerDirectory, - dateIsValid, - immutableAssign, - root, - testImage, - webtorrentAdd -} from '../miscs/miscs' +import { buildAbsoluteFixturePath, buildServerDirectory, dateIsValid, immutableAssign, testImage, webtorrentAdd } from '../miscs/miscs' import { makeGetRequest, makePutBodyRequest, makeUploadRequest } from '../requests/requests' import { waitJobs } from '../server/jobs' import { ServerInfo } from '../server/servers' @@ -335,7 +327,7 @@ async function checkVideoFilesWereRemoved ( ] ) { for (const directory of directories) { - const directoryPath = buildServerDirectory(serverNumber, directory) + const directoryPath = buildServerDirectory({ internalServerNumber: serverNumber }, directory) const directoryExists = await pathExists(directoryPath) if (directoryExists === false) continue @@ -489,7 +481,8 @@ function rateVideo (url: string, accessToken: string, id: number, rating: string function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolution: number) { return new Promise((res, rej) => { const torrentName = videoUUID + '-' + resolution + '.torrent' - const torrentPath = join(root(), 'test' + server.internalServerNumber, 'torrents', torrentName) + const torrentPath = buildServerDirectory(server, join('torrents', torrentName)) + readFile(torrentPath, (err, data) => { if (err) return rej(err) -- 2.41.0