From 40298b02546e8225dd21bf6048fe7f224aefc32a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 2 Oct 2017 12:20:26 +0200 Subject: Implement video transcoding on server side --- config/default.yaml | 9 +- server/controllers/api/videos/index.ts | 9 +- server/helpers/core-utils.ts | 8 +- server/helpers/utils.ts | 27 +++++- server/initializers/constants.ts | 30 ++++-- server/lib/jobs/handlers/index.ts | 6 +- server/lib/jobs/handlers/video-file-optimizer.ts | 78 +++++++++++++++ server/lib/jobs/handlers/video-file-transcoder.ts | 33 +++++++ server/lib/jobs/handlers/video-transcoder.ts | 34 ------- server/models/pod/pod.ts | 2 +- server/models/user/user.ts | 3 +- server/models/video/video-interface.ts | 63 +++---------- server/models/video/video.ts | 105 +++++++++++++++++++-- server/tests/api/multiple-pods.ts | 65 ++++++++----- server/tests/api/video-transcoder.ts | 4 + server/tests/cli/update-host.ts | 30 +++--- server/tests/utils/videos.ts | 5 +- .../remote-video-update-request.model.ts | 2 - shared/models/videos/index.ts | 1 + shared/models/videos/video-resolution.enum.ts | 8 ++ 20 files changed, 369 insertions(+), 153 deletions(-) create mode 100644 server/lib/jobs/handlers/video-file-optimizer.ts create mode 100644 server/lib/jobs/handlers/video-file-transcoder.ts delete mode 100644 server/lib/jobs/handlers/video-transcoder.ts create mode 100644 shared/models/videos/video-resolution.enum.ts diff --git a/config/default.yaml b/config/default.yaml index 4c19a5b2d..b53fa0d5b 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -41,7 +41,14 @@ user: video_quota: -1 # If enabled, the video will be transcoded to mp4 (x264) with "faststart" flag -# Uses a lot of CPU! +# In addition, if some resolutions are enabled the mp4 video file will be transcoded to these new resolutions. +# Uses a lot of CPU and increases storage! transcoding: enabled: false threads: 2 + resolutions: # Only created if the original video has a higher resolution + 240p: true + 360p: true + 480p: true + 720p: true + 1080p: true diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 6fa84c801..14c969ec3 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -39,13 +39,12 @@ import { getFormattedObjects, renamePromise } from '../../../helpers' -import { TagInstance } from '../../../models' -import { VideoCreate, VideoUpdate } from '../../../../shared' +import { TagInstance, VideoInstance } from '../../../models' +import { VideoCreate, VideoUpdate, VideoResolution } from '../../../../shared' import { abuseVideoRouter } from './abuse' import { blacklistRouter } from './blacklist' import { rateVideoRouter } from './rate' -import { VideoInstance } from '../../../models/video/video-interface' const videosRouter = express.Router() @@ -195,7 +194,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil .then(({ author, tagInstances, video }) => { const videoFileData = { extname: extname(videoPhysicalFile.filename), - resolution: 0, // TODO: improve readability, + resolution: VideoResolution.ORIGINAL, size: videoPhysicalFile.size } @@ -230,7 +229,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil } tasks.push( - JobScheduler.Instance.createJob(t, 'videoTranscoder', dataInput) + JobScheduler.Instance.createJob(t, 'videoFileOptimizer', dataInput) ) } diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 2ec7e6515..3118dc500 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -11,7 +11,9 @@ import { rename, unlink, writeFile, - access + access, + stat, + Stats } from 'fs' import * as mkdirp from 'mkdirp' import * as bcrypt from 'bcrypt' @@ -92,6 +94,7 @@ const bcryptGenSaltPromise = promisify1(bcrypt.genSalt) const bcryptHashPromise = promisify2(bcrypt.hash) const createTorrentPromise = promisify2(createTorrent) const rimrafPromise = promisify1WithVoid(rimraf) +const statPromise = promisify1(stat) // --------------------------------------------------------------------------- @@ -115,5 +118,6 @@ export { bcryptGenSaltPromise, bcryptHashPromise, createTorrentPromise, - rimrafPromise + rimrafPromise, + statPromise } diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index ce07ceff9..b74442ab0 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -4,6 +4,7 @@ import * as Promise from 'bluebird' import { pseudoRandomBytesPromise } from './core-utils' import { CONFIG, database as db } from '../initializers' import { ResultList } from '../../shared' +import { VideoResolution } from '../../shared/models/videos/video-resolution.enum' function badRequest (req: express.Request, res: express.Response, next: express.NextFunction) { res.type('json').status(400).end() @@ -13,11 +14,11 @@ function generateRandomString (size: number) { return pseudoRandomBytesPromise(size).then(raw => raw.toString('hex')) } -interface FormatableToJSON { +interface FormattableToJSON { toFormattedJSON () } -function getFormattedObjects (objects: T[], objectsTotal: number) { +function getFormattedObjects (objects: T[], objectsTotal: number) { const formattedObjects: U[] = [] objects.forEach(object => { @@ -47,6 +48,27 @@ function isSignupAllowed () { }) } +function computeResolutionsToTranscode (videoFileHeight: number) { + const resolutionsEnabled: number[] = [] + const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS + + const resolutions = [ + VideoResolution.H_240P, + VideoResolution.H_360P, + VideoResolution.H_480P, + VideoResolution.H_720P, + VideoResolution.H_1080P + ] + + for (const resolution of resolutions) { + if (configResolutions[resolution.toString()] === true && videoFileHeight >= resolution) { + resolutionsEnabled.push(resolution) + } + } + + return resolutionsEnabled +} + type SortType = { sortModel: any, sortValue: string } // --------------------------------------------------------------------------- @@ -56,5 +78,6 @@ export { generateRandomString, getFormattedObjects, isSignupAllowed, + computeResolutionsToTranscode, SortType } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 54e91d35d..073fabd27 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -10,7 +10,8 @@ import { RequestEndpoint, RequestVideoEventType, RequestVideoQaduType, - JobState + JobState, + VideoResolution } from '../../shared/models' // --------------------------------------------------------------------------- @@ -85,7 +86,14 @@ const CONFIG = { }, TRANSCODING: { ENABLED: config.get('transcoding.enabled'), - THREADS: config.get('transcoding.threads') + THREADS: config.get('transcoding.threads'), + RESOLUTIONS: { + '240' : config.get('transcoding.resolutions.240p'), + '360': config.get('transcoding.resolutions.360p'), + '480': config.get('transcoding.resolutions.480p'), + '720': config.get('transcoding.resolutions.720p'), + '1080': config.get('transcoding.resolutions.1080p') + } }, CACHE: { PREVIEWS: { @@ -144,7 +152,7 @@ const VIDEO_CATEGORIES = { 9: 'Comedy', 10: 'Entertainment', 11: 'News', - 12: 'Howto', + 12: 'How To', 13: 'Education', 14: 'Activism', 15: 'Science & Technology', @@ -179,15 +187,17 @@ const VIDEO_LANGUAGES = { 11: 'German', 12: 'Korean', 13: 'French', - 14: 'Italien' + 14: 'Italian' } -const VIDEO_FILE_RESOLUTIONS = { +// TODO: use VideoResolution when https://github.com/Microsoft/TypeScript/issues/13042 is fixed +const VIDEO_FILE_RESOLUTIONS: { [ id: number ]: string } = { 0: 'original', - 1: '360p', - 2: '480p', - 3: '720p', - 4: '1080p' + 240: '240p', + 360: '360p', + 480: '480p', + 720: '720p', + 1080: '1080p' } // --------------------------------------------------------------------------- @@ -202,7 +212,7 @@ const FRIEND_SCORE = { // Number of points we add/remove from a friend after a successful/bad request const PODS_SCORE = { - MALUS: -10, + PENALTY: -10, BONUS: 10 } diff --git a/server/lib/jobs/handlers/index.ts b/server/lib/jobs/handlers/index.ts index 8abddae35..5941427a1 100644 --- a/server/lib/jobs/handlers/index.ts +++ b/server/lib/jobs/handlers/index.ts @@ -1,4 +1,5 @@ -import * as videoTranscoder from './video-transcoder' +import * as videoFileOptimizer from './video-file-optimizer' +import * as videoFileTranscoder from './video-file-transcoder' export interface JobHandler { process (data: object): T @@ -7,7 +8,8 @@ export interface JobHandler { } const jobHandlers: { [ handlerName: string ]: JobHandler } = { - videoTranscoder + videoFileOptimizer, + videoFileTranscoder } export { diff --git a/server/lib/jobs/handlers/video-file-optimizer.ts b/server/lib/jobs/handlers/video-file-optimizer.ts new file mode 100644 index 000000000..a87ce52dc --- /dev/null +++ b/server/lib/jobs/handlers/video-file-optimizer.ts @@ -0,0 +1,78 @@ +import * as Promise from 'bluebird' + +import { database as db } from '../../../initializers/database' +import { logger, computeResolutionsToTranscode } from '../../../helpers' +import { VideoInstance } from '../../../models' +import { addVideoToFriends } from '../../friends' +import { JobScheduler } from '../job-scheduler' + +function process (data: { videoUUID: string }) { + return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => { + return video.optimizeOriginalVideofile().then(() => video) + }) +} + +function onError (err: Error, jobId: number) { + logger.error('Error when optimized video file in job %d.', jobId, err) + return Promise.resolve() +} + +function onSuccess (jobId: number, video: VideoInstance) { + logger.info('Job %d is a success.', jobId) + + video.toAddRemoteJSON() + .then(remoteVideo => { + // Now we'll add the video's meta data to our friends + return addVideoToFriends(remoteVideo, null) + }) + .then(() => { + return video.getOriginalFileHeight() + }) + .then(originalFileHeight => { + // Create transcoding jobs if there are enabled resolutions + const resolutionsEnabled = computeResolutionsToTranscode(originalFileHeight) + logger.info( + 'Resolutions computed for video %s and origin file height of %d.', video.uuid, originalFileHeight, + { resolutions: resolutionsEnabled } + ) + + if (resolutionsEnabled.length === 0) return undefined + + return db.sequelize.transaction(t => { + const tasks: Promise[] = [] + + resolutionsEnabled.forEach(resolution => { + const dataInput = { + videoUUID: video.uuid, + resolution + } + + const p = JobScheduler.Instance.createJob(t, 'videoFileTranscoder', dataInput) + tasks.push(p) + }) + + return Promise.all(tasks).then(() => resolutionsEnabled) + }) + }) + .then(resolutionsEnabled => { + if (resolutionsEnabled === undefined) { + logger.info('No transcoding jobs created for video %s (no resolutions enabled).') + return undefined + } + + logger.info('Transcoding jobs created for uuid %s.', video.uuid, { resolutionsEnabled }) + }) + .catch((err: Error) => { + logger.debug('Cannot transcode the video.', err) + throw err + }) + +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/handlers/video-file-transcoder.ts b/server/lib/jobs/handlers/video-file-transcoder.ts new file mode 100644 index 000000000..0e45b4dca --- /dev/null +++ b/server/lib/jobs/handlers/video-file-transcoder.ts @@ -0,0 +1,33 @@ +import { database as db } from '../../../initializers/database' +import { updateVideoToFriends } from '../../friends' +import { logger } from '../../../helpers' +import { VideoInstance } from '../../../models' +import { VideoResolution } from '../../../../shared' + +function process (data: { videoUUID: string, resolution: VideoResolution }) { + return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => { + return video.transcodeOriginalVideofile(data.resolution).then(() => video) + }) +} + +function onError (err: Error, jobId: number) { + logger.error('Error when transcoding video file in job %d.', jobId, err) + return Promise.resolve() +} + +function onSuccess (jobId: number, video: VideoInstance) { + logger.info('Job %d is a success.', jobId) + + const remoteVideo = video.toUpdateRemoteJSON() + + // Now we'll add the video's meta data to our friends + return updateVideoToFriends(remoteVideo, null) +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/handlers/video-transcoder.ts b/server/lib/jobs/handlers/video-transcoder.ts deleted file mode 100644 index 87d8ffa6a..000000000 --- a/server/lib/jobs/handlers/video-transcoder.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { database as db } from '../../../initializers/database' -import { logger } from '../../../helpers' -import { addVideoToFriends } from '../../../lib' -import { VideoInstance } from '../../../models' - -function process (data: { videoUUID: string }) { - return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID).then(video => { - // TODO: handle multiple resolutions - const videoFile = video.VideoFiles[0] - return video.transcodeVideofile(videoFile).then(() => video) - }) -} - -function onError (err: Error, jobId: number) { - logger.error('Error when transcoding video file in job %d.', jobId, err) - return Promise.resolve() -} - -function onSuccess (jobId: number, video: VideoInstance) { - logger.info('Job %d is a success.', jobId) - - video.toAddRemoteJSON().then(remoteVideo => { - // Now we'll add the video's meta data to our friends - return addVideoToFriends(remoteVideo, null) - }) -} - -// --------------------------------------------------------------------------- - -export { - process, - onError, - onSuccess -} diff --git a/server/models/pod/pod.ts b/server/models/pod/pod.ts index df6412721..1440ac9b4 100644 --- a/server/models/pod/pod.ts +++ b/server/models/pod/pod.ts @@ -219,7 +219,7 @@ updatePodsScore = function (goodPods: number[], badPods: number[]) { } if (badPods.length !== 0) { - incrementScores(badPods, PODS_SCORE.MALUS) + incrementScores(badPods, PODS_SCORE.PENALTY) .then(() => removeBadPods()) .catch(err => { if (err) logger.error('Cannot decrement scores of bad pods.', err) diff --git a/server/models/user/user.ts b/server/models/user/user.ts index 79a595528..7a21dbefa 100644 --- a/server/models/user/user.ts +++ b/server/models/user/user.ts @@ -12,6 +12,7 @@ import { isUserDisplayNSFWValid, isUserVideoQuotaValid } from '../../helpers' +import { VideoResolution } from '../../../shared' import { addMethodsToModel } from '../utils' import { @@ -245,7 +246,7 @@ function getOriginalVideoFileTotalFromUser (user: UserInstance) { // attributes = [] because we don't want other fields than the sum const query = { where: { - resolution: 0 // Original, TODO: improve readability + resolution: VideoResolution.ORIGINAL }, include: [ { diff --git a/server/models/video/video-interface.ts b/server/models/video/video-interface.ts index fb31c6a8f..340426f45 100644 --- a/server/models/video/video-interface.ts +++ b/server/models/video/video-interface.ts @@ -7,60 +7,17 @@ import { VideoFileAttributes, VideoFileInstance } from './video-file-interface' // Don't use barrel, import just what we need import { Video as FormattedVideo } from '../../../shared/models/videos/video.model' +import { RemoteVideoUpdateData } from '../../../shared/models/pods/remote-video/remote-video-update-request.model' +import { RemoteVideoCreateData } from '../../../shared/models/pods/remote-video/remote-video-create-request.model' import { ResultList } from '../../../shared/models/result-list.model' -export type FormattedRemoteVideoFile = { - infoHash: string - resolution: number - extname: string - size: number -} - -export type FormattedAddRemoteVideo = { - uuid: string - name: string - category: number - licence: number - language: number - nsfw: boolean - description: string - author: string - duration: number - thumbnailData: string - tags: string[] - createdAt: Date - updatedAt: Date - views: number - likes: number - dislikes: number - files: FormattedRemoteVideoFile[] -} - -export type FormattedUpdateRemoteVideo = { - uuid: string - name: string - category: number - licence: number - language: number - nsfw: boolean - description: string - author: string - duration: number - tags: string[] - createdAt: Date - updatedAt: Date - views: number - likes: number - dislikes: number - files: FormattedRemoteVideoFile[] -} - export namespace VideoMethods { export type GetThumbnailName = (this: VideoInstance) => string export type GetPreviewName = (this: VideoInstance) => string export type IsOwned = (this: VideoInstance) => boolean export type ToFormattedJSON = (this: VideoInstance) => FormattedVideo + export type GetOriginalFile = (this: VideoInstance) => VideoFileInstance export type GenerateMagnetUri = (this: VideoInstance, videoFile: VideoFileInstance) => string export type GetTorrentFileName = (this: VideoInstance, videoFile: VideoFileInstance) => string export type GetVideoFilename = (this: VideoInstance, videoFile: VideoFileInstance) => string @@ -69,10 +26,12 @@ export namespace VideoMethods { export type GetVideoFilePath = (this: VideoInstance, videoFile: VideoFileInstance) => string export type CreateTorrentAndSetInfoHash = (this: VideoInstance, videoFile: VideoFileInstance) => Promise - export type ToAddRemoteJSON = (this: VideoInstance) => Promise - export type ToUpdateRemoteJSON = (this: VideoInstance) => FormattedUpdateRemoteVideo + export type ToAddRemoteJSON = (this: VideoInstance) => Promise + export type ToUpdateRemoteJSON = (this: VideoInstance) => RemoteVideoUpdateData - export type TranscodeVideofile = (this: VideoInstance, inputVideoFile: VideoFileInstance) => Promise + export type OptimizeOriginalVideofile = (this: VideoInstance) => Promise + export type TranscodeOriginalVideofile = (this: VideoInstance, resolution: number) => Promise + export type GetOriginalFileHeight = (this: VideoInstance) => Promise // Return thumbnail name export type GenerateThumbnailFromData = (video: VideoInstance, thumbnailData: string) => Promise @@ -147,6 +106,7 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In createPreview: VideoMethods.CreatePreview createThumbnail: VideoMethods.CreateThumbnail createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash + getOriginalFile: VideoMethods.GetOriginalFile generateMagnetUri: VideoMethods.GenerateMagnetUri getPreviewName: VideoMethods.GetPreviewName getThumbnailName: VideoMethods.GetThumbnailName @@ -161,9 +121,12 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In toAddRemoteJSON: VideoMethods.ToAddRemoteJSON toFormattedJSON: VideoMethods.ToFormattedJSON toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON - transcodeVideofile: VideoMethods.TranscodeVideofile + optimizeOriginalVideofile: VideoMethods.OptimizeOriginalVideofile + transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile + getOriginalFileHeight: VideoMethods.GetOriginalFileHeight setTags: Sequelize.HasManySetAssociationsMixin + addVideoFile: Sequelize.HasManyAddAssociationMixin setVideoFiles: Sequelize.HasManySetAssociationsMixin } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index e011c3b4d..28df91a7b 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -22,7 +22,8 @@ import { unlinkPromise, renamePromise, writeFilePromise, - createTorrentPromise + createTorrentPromise, + statPromise } from '../../helpers' import { CONFIG, @@ -35,7 +36,8 @@ import { VIDEO_FILE_RESOLUTIONS } from '../../initializers' import { removeVideoToFriends } from '../../lib' -import { VideoFileInstance } from './video-file-interface' +import { VideoResolution } from '../../../shared' +import { VideoFileInstance, VideoFileModel } from './video-file-interface' import { addMethodsToModel, getSort } from '../utils' import { @@ -46,6 +48,7 @@ import { } from './video-interface' let Video: Sequelize.Model +let getOriginalFile: VideoMethods.GetOriginalFile let generateMagnetUri: VideoMethods.GenerateMagnetUri let getVideoFilename: VideoMethods.GetVideoFilename let getThumbnailName: VideoMethods.GetThumbnailName @@ -55,11 +58,13 @@ let isOwned: VideoMethods.IsOwned let toFormattedJSON: VideoMethods.ToFormattedJSON let toAddRemoteJSON: VideoMethods.ToAddRemoteJSON let toUpdateRemoteJSON: VideoMethods.ToUpdateRemoteJSON -let transcodeVideofile: VideoMethods.TranscodeVideofile +let optimizeOriginalVideofile: VideoMethods.OptimizeOriginalVideofile +let transcodeOriginalVideofile: VideoMethods.TranscodeOriginalVideofile let createPreview: VideoMethods.CreatePreview let createThumbnail: VideoMethods.CreateThumbnail let getVideoFilePath: VideoMethods.GetVideoFilePath let createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash +let getOriginalFileHeight: VideoMethods.GetOriginalFileHeight let generateThumbnailFromData: VideoMethods.GenerateThumbnailFromData let getDurationFromFile: VideoMethods.GetDurationFromFile @@ -251,6 +256,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da getTorrentFileName, getVideoFilename, getVideoFilePath, + getOriginalFile, isOwned, removeFile, removePreview, @@ -259,7 +265,9 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da toAddRemoteJSON, toFormattedJSON, toUpdateRemoteJSON, - transcodeVideofile + optimizeOriginalVideofile, + transcodeOriginalVideofile, + getOriginalFileHeight ] addMethodsToModel(Video, classMethods, instanceMethods) @@ -327,9 +335,14 @@ function afterDestroy (video: VideoInstance, options: { transaction: Sequelize.T return Promise.all(tasks) } +getOriginalFile = function (this: VideoInstance) { + if (Array.isArray(this.VideoFiles) === false) return undefined + + return this.VideoFiles.find(file => file.resolution === VideoResolution.ORIGINAL) +} + getVideoFilename = function (this: VideoInstance, videoFile: VideoFileInstance) { - // return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + videoFile.extname - return this.uuid + videoFile.extname + return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + videoFile.extname } getThumbnailName = function (this: VideoInstance) { @@ -345,8 +358,7 @@ getPreviewName = function (this: VideoInstance) { getTorrentFileName = function (this: VideoInstance, videoFile: VideoFileInstance) { const extension = '.torrent' - // return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + extension - return this.uuid + extension + return this.uuid + '-' + VIDEO_FILE_RESOLUTIONS[videoFile.resolution] + extension } isOwned = function (this: VideoInstance) { @@ -552,9 +564,10 @@ toUpdateRemoteJSON = function (this: VideoInstance) { return json } -transcodeVideofile = function (this: VideoInstance, inputVideoFile: VideoFileInstance) { +optimizeOriginalVideofile = function (this: VideoInstance) { const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR const newExtname = '.mp4' + const inputVideoFile = this.getOriginalFile() const videoInputPath = join(videosDirectory, this.getVideoFilename(inputVideoFile)) const videoOutputPath = join(videosDirectory, this.id + '-transcoded' + newExtname) @@ -574,6 +587,12 @@ transcodeVideofile = function (this: VideoInstance, inputVideoFile: VideoFileIns return renamePromise(videoOutputPath, this.getVideoFilePath(inputVideoFile)) }) + .then(() => { + return statPromise(this.getVideoFilePath(inputVideoFile)) + }) + .then(stats => { + return inputVideoFile.set('size', stats.size) + }) .then(() => { return this.createTorrentAndSetInfoHash(inputVideoFile) }) @@ -594,6 +613,74 @@ transcodeVideofile = function (this: VideoInstance, inputVideoFile: VideoFileIns }) } +transcodeOriginalVideofile = function (this: VideoInstance, resolution: VideoResolution) { + const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR + const extname = '.mp4' + + // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed + const videoInputPath = join(videosDirectory, this.getVideoFilename(this.getOriginalFile())) + + const newVideoFile = (Video['sequelize'].models.VideoFile as VideoFileModel).build({ + resolution, + extname, + size: 0, + videoId: this.id + }) + const videoOutputPath = join(videosDirectory, this.getVideoFilename(newVideoFile)) + const resolutionWidthSizes = { + 1: '240x?', + 2: '360x?', + 3: '480x?', + 4: '720x?', + 5: '1080x?' + } + + return new Promise((res, rej) => { + ffmpeg(videoInputPath) + .output(videoOutputPath) + .videoCodec('libx264') + .size(resolutionWidthSizes[resolution]) + .outputOption('-threads ' + CONFIG.TRANSCODING.THREADS) + .outputOption('-movflags faststart') + .on('error', rej) + .on('end', () => { + return statPromise(videoOutputPath) + .then(stats => { + newVideoFile.set('size', stats.size) + + return undefined + }) + .then(() => { + return this.createTorrentAndSetInfoHash(newVideoFile) + }) + .then(() => { + return newVideoFile.save() + }) + .then(() => { + return this.VideoFiles.push(newVideoFile) + }) + .then(() => { + return res() + }) + .catch(rej) + }) + .run() + }) +} + +getOriginalFileHeight = function (this: VideoInstance) { + const originalFilePath = this.getVideoFilePath(this.getOriginalFile()) + + return new Promise((res, rej) => { + ffmpeg.ffprobe(originalFilePath, (err, metadata) => { + if (err) return rej(err) + + const videoStream = metadata.streams.find(s => s.codec_type === 'video') + return res(videoStream.height) + }) + }) +} + removeThumbnail = function (this: VideoInstance) { const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) return unlinkPromise(thumbnailPath) diff --git a/server/tests/api/multiple-pods.ts b/server/tests/api/multiple-pods.ts index 7117ab290..9860935e5 100644 --- a/server/tests/api/multiple-pods.ts +++ b/server/tests/api/multiple-pods.ts @@ -129,7 +129,7 @@ describe('Test multiple pods', function () { }) it('Should upload the video on pod 2 and propagate on each pod', async function () { - this.timeout(60000) + this.timeout(120000) const videoAttributes = { name: 'my super name for pod 2', @@ -143,12 +143,12 @@ describe('Test multiple pods', function () { } await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) - // Transcoding, so wait more that 22 seconds - await wait(42000) + // Transcoding, so wait more than 22000 + await wait(60000) // All pods should have this video for (const server of servers) { - let baseMagnet = null + let baseMagnet = {} const res = await getVideosList(server.url) @@ -172,28 +172,51 @@ describe('Test multiple pods', function () { expect(dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') - expect(video.files).to.have.lengthOf(1) + expect(video.files).to.have.lengthOf(5) - const file = video.files[0] - const magnetUri = file.magnetUri - expect(file.magnetUri).to.have.lengthOf.above(2) - expect(file.resolution).to.equal(0) - expect(file.resolutionLabel).to.equal('original') - expect(file.size).to.equal(942961) + // Check common attributes + for (const file of video.files) { + expect(file.magnetUri).to.have.lengthOf.above(2) - if (server.url !== 'http://localhost:9002') { - expect(video.isLocal).to.be.false - } else { - expect(video.isLocal).to.be.true - } + if (server.url !== 'http://localhost:9002') { + expect(video.isLocal).to.be.false + } else { + expect(video.isLocal).to.be.true + } - // All pods should have the same magnet Uri - if (baseMagnet === null) { - baseMagnet = magnetUri - } else { - expect(baseMagnet).to.equal(magnetUri) + // All pods should have the same magnet Uri + if (baseMagnet[file.resolution] === undefined) { + baseMagnet[file.resolution] = file.magnet + } else { + expect(baseMagnet[file.resolution]).to.equal(file.magnet) + } } + const originalFile = video.files.find(f => f.resolution === 0) + expect(originalFile).not.to.be.undefined + expect(originalFile.resolutionLabel).to.equal('original') + expect(originalFile.size).to.equal(711327) + + const file240p = video.files.find(f => f.resolution === 1) + expect(file240p).not.to.be.undefined + expect(file240p.resolutionLabel).to.equal('240p') + expect(file240p.size).to.equal(139953) + + const file360p = video.files.find(f => f.resolution === 2) + expect(file360p).not.to.be.undefined + expect(file360p.resolutionLabel).to.equal('360p') + expect(file360p.size).to.equal(169926) + + const file480p = video.files.find(f => f.resolution === 3) + expect(file480p).not.to.be.undefined + expect(file480p.resolutionLabel).to.equal('480p') + expect(file480p.size).to.equal(206758) + + const file720p = video.files.find(f => f.resolution === 4) + expect(file720p).not.to.be.undefined + expect(file720p.resolutionLabel).to.equal('720p') + expect(file720p.size).to.equal(314913) + const test = await testVideoImage(server.url, 'video_short2.webm', video.thumbnailPath) expect(test).to.equal(true) } diff --git a/server/tests/api/video-transcoder.ts b/server/tests/api/video-transcoder.ts index c6d4c61f5..b5d84d9e7 100644 --- a/server/tests/api/video-transcoder.ts +++ b/server/tests/api/video-transcoder.ts @@ -42,6 +42,8 @@ describe('Test video transcoding', function () { const res = await getVideosList(servers[0].url) const video = res.body.data[0] + expect(video.files).to.have.lengthOf(1) + const magnetUri = video.files[0].magnetUri expect(magnetUri).to.match(/\.webm/) @@ -66,6 +68,8 @@ describe('Test video transcoding', function () { const res = await getVideosList(servers[1].url) const video = res.body.data[0] + expect(video.files).to.have.lengthOf(5) + const magnetUri = video.files[0].magnetUri expect(magnetUri).to.match(/\.mp4/) diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts index 644b3807e..e31a84156 100644 --- a/server/tests/cli/update-host.ts +++ b/server/tests/cli/update-host.ts @@ -12,14 +12,15 @@ import { runServer, ServerInfo, setAccessTokensToServers, - uploadVideo + uploadVideo, + wait } from '../utils' describe('Test update host scripts', function () { let server: ServerInfo before(async function () { - this.timeout(30000) + this.timeout(60000) await flushTests() @@ -28,36 +29,43 @@ describe('Test update host scripts', function () { port: 9256 } } - server = await runServer(1, overrideConfig) + // Run server 2 to have transcoding enabled + server = await runServer(2, overrideConfig) await setAccessTokensToServers([ server ]) // Upload two videos for our needs const videoAttributes = {} await uploadVideo(server.url, server.accessToken, videoAttributes) await uploadVideo(server.url, server.accessToken, videoAttributes) + await wait(30000) }) it('Should update torrent hosts', async function () { this.timeout(30000) killallServers([ server ]) - server = await runServer(1) + // Run server with standard configuration + server = await runServer(2) const env = getEnvCli(server) await execCLI(`${env} npm run update-host`) const res = await getVideosList(server.url) const videos = res.body.data + expect(videos).to.have.lengthOf(2) - expect(videos[0].files[0].magnetUri).to.contain('localhost%3A9001%2Ftracker%2Fsocket') - expect(videos[0].files[0].magnetUri).to.contain('localhost%3A9001%2Fstatic%2Fwebseed%2F') + for (const video of videos) { + expect(video.files).to.have.lengthOf(5) - expect(videos[1].files[0].magnetUri).to.contain('localhost%3A9001%2Ftracker%2Fsocket') - expect(videos[1].files[0].magnetUri).to.contain('localhost%3A9001%2Fstatic%2Fwebseed%2F') + for (const file of video.files) { + expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket') + expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F') - const torrent = await parseTorrentVideo(server, videos[0].uuid) - expect(torrent.announce[0]).to.equal('ws://localhost:9001/tracker/socket') - expect(torrent.urlList[0]).to.contain('http://localhost:9001/static/webseed') + const torrent = await parseTorrentVideo(server, video.uuid, file.resolutionLabel) + expect(torrent.announce[0]).to.equal('ws://localhost:9002/tracker/socket') + expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed') + } + } }) after(async function () { diff --git a/server/tests/utils/videos.ts b/server/tests/utils/videos.ts index 0de506cd9..7f8bd39c0 100644 --- a/server/tests/utils/videos.ts +++ b/server/tests/utils/videos.ts @@ -238,9 +238,10 @@ function rateVideo (url: string, accessToken: string, id: number, rating: string .expect(specialStatus) } -function parseTorrentVideo (server: ServerInfo, videoUUID: string) { +function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolutionLabel: string) { return new Promise((res, rej) => { - const torrentPath = join(__dirname, '..', '..', '..', 'test' + server.serverNumber, 'torrents', videoUUID + '.torrent') + const torrentName = videoUUID + '-' + resolutionLabel + '.torrent' + const torrentPath = join(__dirname, '..', '..', '..', 'test' + server.serverNumber, 'torrents', torrentName) readFile(torrentPath, (err, data) => { if (err) return rej(err) diff --git a/shared/models/pods/remote-video/remote-video-update-request.model.ts b/shared/models/pods/remote-video/remote-video-update-request.model.ts index dd3e2ae1a..7f34a30ae 100644 --- a/shared/models/pods/remote-video/remote-video-update-request.model.ts +++ b/shared/models/pods/remote-video/remote-video-update-request.model.ts @@ -2,8 +2,6 @@ export interface RemoteVideoUpdateData { uuid: string tags: string[] name: string - extname: string - infoHash: string category: number licence: number language: number diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts index 64d1676c5..35144dbad 100644 --- a/shared/models/videos/index.ts +++ b/shared/models/videos/index.ts @@ -6,5 +6,6 @@ export * from './video-abuse.model' export * from './video-blacklist.model' export * from './video-create.model' export * from './video-rate.type' +export * from './video-resolution.enum' export * from './video-update.model' export * from './video.model' diff --git a/shared/models/videos/video-resolution.enum.ts b/shared/models/videos/video-resolution.enum.ts new file mode 100644 index 000000000..bdce77ed6 --- /dev/null +++ b/shared/models/videos/video-resolution.enum.ts @@ -0,0 +1,8 @@ +export enum VideoResolution { + ORIGINAL = 0, + H_240P = 240, + H_360P = 360, + H_480P = 480, + H_720P = 720, + H_1080P = 1080 +} -- cgit v1.2.3