From: Chocobozzz Date: Thu, 18 Oct 2018 14:53:52 +0000 (+0200) Subject: Fix optimize old videos script X-Git-Tag: v1.1.0-alpha.2~13 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=74cd011b6c089cac88a4b8aa76ad3be2ca4f1c15;p=github%2FChocobozzz%2FPeerTube.git Fix optimize old videos script --- diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index 02026b3da..c93f82316 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts @@ -1,8 +1,10 @@ -import { VIDEO_TRANSCODING_FPS } from '../server/initializers/constants' +import { CONFIG, VIDEO_TRANSCODING_FPS } from '../server/initializers/constants' import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffmpeg-utils' import { getMaxBitrate } from '../shared/models/videos' import { VideoModel } from '../server/models/video/video' import { optimizeVideofile } from '../server/lib/video-transcoding' +import { initDatabaseModels } from '../server/initializers' +import { join } from 'path' run() .then(() => process.exit(0)) @@ -12,11 +14,13 @@ run() }) async function run () { + await initDatabaseModels(true) + const localVideos = await VideoModel.listLocal() for (const video of localVideos) { for (const file of video.VideoFiles) { - const inputPath = video.getVideoFilename(file) + const inputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) const [ videoBitrate, fps, resolution ] = await Promise.all([ getVideoFileBitrate(inputPath), diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 0a567873c..85795d2ed 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -18,7 +18,8 @@ import { ServerInfo, setAccessTokensToServers, uploadVideo, - webtorrentAdd + webtorrentAdd, + generateHighBitrateVideo } from '../../utils' import { join } from 'path' import { waitJobs } from '../../utils/server/jobs' @@ -283,29 +284,13 @@ describe('Test video transcoding', function () { } }) - const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) it('Should respect maximum bitrate values', async function () { this.timeout(160000) + let tempFixturePath: string + { - const exists = await pathExists(tempFixturePath) - if (!exists) { - - // Generate a random, high bitrate video on the fly, so we don't have to include - // a large file in the repo. The video needs to have a certain minimum length so - // that FFmpeg properly applies bitrate limits. - // https://stackoverflow.com/a/15795112 - await new Promise(async (res, rej) => { - ffmpeg() - .outputOptions([ '-f rawvideo', '-video_size 1920x1080', '-i /dev/urandom' ]) - .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) - .outputOptions([ '-maxrate 10M', '-bufsize 10M' ]) - .output(tempFixturePath) - .on('error', rej) - .on('end', res) - .run() - }) - } + tempFixturePath = await generateHighBitrateVideo() const bitrate = await getVideoFileBitrate(tempFixturePath) expect(bitrate).to.be.above(getMaxBitrate(VideoResolution.H_1080P, 60, VIDEO_TRANSCODING_FPS)) diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts index 6201314ce..8f8acf56c 100644 --- a/server/tests/cli/index.ts +++ b/server/tests/cli/index.ts @@ -4,3 +4,4 @@ import './create-transcoding-job' import './peertube' import './reset-password' import './update-host' +import './update-host' diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts new file mode 100644 index 000000000..66dd39cce --- /dev/null +++ b/server/tests/cli/optimize-old-videos.ts @@ -0,0 +1,120 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' +import * as chai from 'chai' +import { getMaxBitrate, Video, VideoDetails, VideoResolution } from '../../../shared/models/videos' +import { + doubleFollow, + execCLI, + flushAndRunMultipleServers, + flushTests, generateHighBitrateVideo, + getEnvCli, + getVideo, + getVideosList, + killallServers, root, + ServerInfo, + setAccessTokensToServers, + uploadVideo, viewVideo, wait +} from '../utils' +import { waitJobs } from '../utils/server/jobs' +import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../../helpers/ffmpeg-utils' +import { VIDEO_TRANSCODING_FPS } from '../../initializers' +import { join } from 'path' + +const expect = chai.expect + +describe('Test optimize old videos', function () { + let servers: ServerInfo[] = [] + let video1UUID: string + let video2UUID: string + + before(async function () { + this.timeout(200000) + + await flushTests() + + // Run server 2 to have transcoding enabled + servers = await flushAndRunMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + let tempFixturePath: string + + { + tempFixturePath = await generateHighBitrateVideo() + + const bitrate = await getVideoFileBitrate(tempFixturePath) + expect(bitrate).to.be.above(getMaxBitrate(VideoResolution.H_1080P, 60, VIDEO_TRANSCODING_FPS)) + } + + // Upload two videos for our needs + const res1 = await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video1', fixture: tempFixturePath }) + video1UUID = res1.body.video.uuid + const res2 = await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video2', fixture: tempFixturePath }) + video2UUID = res2.body.video.uuid + + await waitJobs(servers) + }) + + it('Should have two video files on each server', async function () { + this.timeout(30000) + + for (const server of servers) { + const res = await getVideosList(server.url) + const videos = res.body.data + expect(videos).to.have.lengthOf(2) + + for (const video of videos) { + const res2 = await getVideo(server.url, video.uuid) + const videoDetail: VideoDetails = res2.body + expect(videoDetail.files).to.have.lengthOf(1) + } + } + }) + + it('Should run optimize script', async function () { + this.timeout(120000) + + const env = getEnvCli(servers[0]) + await execCLI(`${env} npm run optimize-old-videos`) + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideosList(server.url) + const videos: Video[] = res.body.data + + expect(videos).to.have.lengthOf(2) + + for (const video of videos) { + await viewVideo(server.url, video.uuid) + + // Refresh video + await waitJobs(servers) + await wait(5000) + await waitJobs(servers) + + const res2 = await getVideo(server.url, video.uuid) + const videosDetails: VideoDetails = res2.body + + expect(videosDetails.files).to.have.lengthOf(1) + const file = videosDetails.files[0] + + expect(file.size).to.be.below(5000000) + + const path = join(root(), 'test1', 'videos', video.uuid + '-' + file.resolution.id + '.mp4') + const bitrate = await getVideoFileBitrate(path) + const fps = await getVideoFileFPS(path) + const resolution = await getVideoFileResolution(path) + + expect(resolution.videoFileResolution).to.equal(file.resolution.id) + expect(bitrate).to.be.below(getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) + } + } + }) + + after(async function () { + killallServers(servers) + }) +}) diff --git a/server/tests/utils/miscs/miscs.ts b/server/tests/utils/miscs/miscs.ts index d20fa96b8..589daa420 100644 --- a/server/tests/utils/miscs/miscs.ts +++ b/server/tests/utils/miscs/miscs.ts @@ -4,7 +4,8 @@ import * as chai from 'chai' import { isAbsolute, join } from 'path' import * as request from 'supertest' import * as WebTorrent from 'webtorrent' -import { readFile } from 'fs-extra' +import { pathExists, readFile } from 'fs-extra' +import * as ffmpeg from 'fluent-ffmpeg' const expect = chai.expect let webtorrent = new WebTorrent() @@ -61,6 +62,31 @@ function buildAbsoluteFixturePath (path: string, customTravisPath = false) { return join(__dirname, '..', '..', 'fixtures', path) } +async function generateHighBitrateVideo () { + const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) + + const exists = await pathExists(tempFixturePath) + if (!exists) { + + // Generate a random, high bitrate video on the fly, so we don't have to include + // a large file in the repo. The video needs to have a certain minimum length so + // that FFmpeg properly applies bitrate limits. + // https://stackoverflow.com/a/15795112 + return new Promise(async (res, rej) => { + ffmpeg() + .outputOptions([ '-f rawvideo', '-video_size 1920x1080', '-i /dev/urandom' ]) + .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) + .outputOptions([ '-maxrate 10M', '-bufsize 10M' ]) + .output(tempFixturePath) + .on('error', rej) + .on('end', () => res(tempFixturePath)) + .run() + }) + } + + return tempFixturePath +} + // --------------------------------------------------------------------------- export { @@ -70,5 +96,6 @@ export { immutableAssign, testImage, buildAbsoluteFixturePath, - root + root, + generateHighBitrateVideo }