From 14e2014acc1362cfbb770c051a7254b156cd8efb Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Dec 2018 14:52:50 +0100 Subject: Support additional video extensions --- server/controllers/api/config.ts | 5 +- server/controllers/api/users/me.ts | 4 +- server/controllers/api/video-channel.ts | 4 +- server/controllers/api/videos/captions.ts | 4 +- server/controllers/api/videos/import.ts | 11 +-- server/controllers/api/videos/index.ts | 12 ++- server/helpers/custom-validators/video-captions.ts | 4 +- server/helpers/custom-validators/video-imports.ts | 4 +- server/helpers/custom-validators/videos.ts | 13 ++- server/initializers/checker-before-init.ts | 2 +- server/initializers/constants.ts | 96 ++++++++++++++------- .../migrations/0295-video-file-extname.ts | 49 +++++++++++ server/lib/activitypub/actor.ts | 7 +- server/lib/activitypub/videos.ts | 7 +- server/models/redundancy/video-redundancy.ts | 6 +- server/models/video/video-file.ts | 4 +- server/models/video/video-format-utils.ts | 6 +- server/tests/api/check-params/config.ts | 1 + server/tests/api/check-params/videos.ts | 7 +- server/tests/api/server/config.ts | 30 ++++++- server/tests/api/videos/video-transcoder.ts | 31 ++++++- server/tests/fixtures/video_short.avi | Bin 0 -> 584656 bytes server/tests/fixtures/video_short.mkv | Bin 0 -> 40642 bytes 23 files changed, 224 insertions(+), 83 deletions(-) create mode 100644 server/initializers/migrations/0295-video-file-extname.ts create mode 100644 server/tests/fixtures/video_short.avi create mode 100644 server/tests/fixtures/video_short.mkv (limited to 'server') diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index d65e321e9..c75002aaf 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -172,7 +172,8 @@ async function updateCustomConfig (req: express.Request, res: express.Response, 'instance.defaultClientRoute', 'instance.shortDescription', 'cache.videoCaptions', - 'signup.requiresEmailVerification' + 'signup.requiresEmailVerification', + 'transcoding.allowAdditionalExtensions' ) toUpdateJSON.user['video_quota'] = toUpdate.user.videoQuota toUpdateJSON.user['video_quota_daily'] = toUpdate.user.videoQuotaDaily @@ -180,6 +181,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response, toUpdateJSON.instance['short_description'] = toUpdate.instance.shortDescription toUpdateJSON.instance['default_nsfw_policy'] = toUpdate.instance.defaultNSFWPolicy toUpdateJSON.signup['requires_email_verification'] = toUpdate.signup.requiresEmailVerification + toUpdateJSON.transcoding['allow_additional_extensions'] = toUpdate.transcoding.allowAdditionalExtensions await writeJSON(CONFIG.CUSTOM_FILE, toUpdateJSON, { spaces: 2 }) @@ -247,6 +249,7 @@ function customConfig (): CustomConfig { }, transcoding: { enabled: CONFIG.TRANSCODING.ENABLED, + allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS, threads: CONFIG.TRANSCODING.THREADS, resolutions: { '240p': CONFIG.TRANSCODING.RESOLUTIONS[ '240p' ], diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index d2456346b..f712b0f0b 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -2,7 +2,7 @@ import * as express from 'express' import 'multer' import { UserUpdateMe, UserVideoRate as FormattedUserVideoRate } from '../../../../shared' import { getFormattedObjects } from '../../../helpers/utils' -import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../../initializers' +import { CONFIG, MIMETYPES, sequelizeTypescript } from '../../../initializers' import { sendUpdateActor } from '../../../lib/activitypub/send' import { asyncMiddleware, @@ -42,7 +42,7 @@ import { AccountModel } from '../../../models/account/account' const auditLogger = auditLoggerFactory('users-me') -const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) +const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) const meRouter = express.Router() diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index fd143a139..3d6a6af7f 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -22,7 +22,7 @@ import { createVideoChannel } from '../../lib/video-channel' import { buildNSFWFilter, createReqFiles, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { setAsyncActorKeys } from '../../lib/activitypub' import { AccountModel } from '../../models/account/account' -import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' +import { CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers' import { logger } from '../../helpers/logger' import { VideoModel } from '../../models/video/video' import { updateAvatarValidator } from '../../middlewares/validators/avatar' @@ -32,7 +32,7 @@ import { resetSequelizeInstance } from '../../helpers/database-utils' import { UserModel } from '../../models/account/user' const auditLogger = auditLoggerFactory('channels') -const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) +const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) const videoChannelRouter = express.Router() diff --git a/server/controllers/api/videos/captions.ts b/server/controllers/api/videos/captions.ts index 3ba918189..9b3661368 100644 --- a/server/controllers/api/videos/captions.ts +++ b/server/controllers/api/videos/captions.ts @@ -2,7 +2,7 @@ import * as express from 'express' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate } from '../../../middlewares' import { addVideoCaptionValidator, deleteVideoCaptionValidator, listVideoCaptionsValidator } from '../../../middlewares/validators' import { createReqFiles } from '../../../helpers/express-utils' -import { CONFIG, sequelizeTypescript, VIDEO_CAPTIONS_MIMETYPE_EXT } from '../../../initializers' +import { CONFIG, MIMETYPES, sequelizeTypescript } from '../../../initializers' import { getFormattedObjects } from '../../../helpers/utils' import { VideoCaptionModel } from '../../../models/video/video-caption' import { VideoModel } from '../../../models/video/video' @@ -12,7 +12,7 @@ import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' const reqVideoCaptionAdd = createReqFiles( [ 'captionfile' ], - VIDEO_CAPTIONS_MIMETYPE_EXT, + MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT, { captionfile: CONFIG.STORAGE.CAPTIONS_DIR } diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index f27d648c7..099ab7b8d 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -3,14 +3,7 @@ import * as magnetUtil from 'magnet-uri' import 'multer' import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' -import { - CONFIG, - IMAGE_MIMETYPE_EXT, - PREVIEWS_SIZE, - sequelizeTypescript, - THUMBNAILS_SIZE, - TORRENT_MIMETYPE_EXT -} from '../../../initializers' +import { CONFIG, MIMETYPES, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers' import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' import { createReqFiles } from '../../../helpers/express-utils' import { logger } from '../../../helpers/logger' @@ -35,7 +28,7 @@ const videoImportsRouter = express.Router() const reqVideoFileImport = createReqFiles( [ 'thumbnailfile', 'previewfile', 'torrentfile' ], - Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT), + Object.assign({}, MIMETYPES.TORRENT.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT), { thumbnailfile: CONFIG.STORAGE.TMP_DIR, previewfile: CONFIG.STORAGE.TMP_DIR, diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 4e4697ef4..00a1302d1 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -7,15 +7,13 @@ import { logger } from '../../../helpers/logger' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { getFormattedObjects, getServerActor } from '../../../helpers/utils' import { - CONFIG, - IMAGE_MIMETYPE_EXT, + CONFIG, MIMETYPES, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, - VIDEO_MIMETYPE_EXT, VIDEO_PRIVACIES } from '../../../initializers' import { @@ -57,7 +55,7 @@ import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-u import { videoCaptionsRouter } from './captions' import { videoImportsRouter } from './import' import { resetSequelizeInstance } from '../../../helpers/database-utils' -import { rename } from 'fs-extra' +import { move } from 'fs-extra' import { watchingRouter } from './watching' const auditLogger = auditLoggerFactory('videos') @@ -65,7 +63,7 @@ const videosRouter = express.Router() const reqVideoFileAdd = createReqFiles( [ 'videofile', 'thumbnailfile', 'previewfile' ], - Object.assign({}, VIDEO_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT), + Object.assign({}, MIMETYPES.VIDEO.MIMETYPE_EXT, MIMETYPES.IMAGE.MIMETYPE_EXT), { videofile: CONFIG.STORAGE.TMP_DIR, thumbnailfile: CONFIG.STORAGE.TMP_DIR, @@ -74,7 +72,7 @@ const reqVideoFileAdd = createReqFiles( ) const reqVideoFileUpdate = createReqFiles( [ 'thumbnailfile', 'previewfile' ], - IMAGE_MIMETYPE_EXT, + MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR, previewfile: CONFIG.STORAGE.TMP_DIR @@ -208,7 +206,7 @@ async function addVideo (req: express.Request, res: express.Response) { // Move physical file const videoDir = CONFIG.STORAGE.VIDEOS_DIR const destination = join(videoDir, video.getVideoFilename(videoFile)) - await rename(videoPhysicalFile.path, destination) + await move(videoPhysicalFile.path, destination) // This is important in case if there is another attempt in the retry process videoPhysicalFile.filename = video.getVideoFilename(videoFile) videoPhysicalFile.path = destination diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts index 177e9e86e..b33d90e18 100644 --- a/server/helpers/custom-validators/video-captions.ts +++ b/server/helpers/custom-validators/video-captions.ts @@ -1,4 +1,4 @@ -import { CONSTRAINTS_FIELDS, VIDEO_CAPTIONS_MIMETYPE_EXT, VIDEO_LANGUAGES } from '../../initializers' +import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers' import { exists, isFileValid } from './misc' import { Response } from 'express' import { VideoModel } from '../../models/video/video' @@ -8,7 +8,7 @@ function isVideoCaptionLanguageValid (value: any) { return exists(value) && VIDEO_LANGUAGES[ value ] !== undefined } -const videoCaptionTypes = Object.keys(VIDEO_CAPTIONS_MIMETYPE_EXT) +const videoCaptionTypes = Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) .concat([ 'application/octet-stream' ]) // MacOS sends application/octet-stream >< .map(m => `(${m})`) const videoCaptionTypesRegex = videoCaptionTypes.join('|') diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts index 4d6ab1fa4..ce9e9193c 100644 --- a/server/helpers/custom-validators/video-imports.ts +++ b/server/helpers/custom-validators/video-imports.ts @@ -1,7 +1,7 @@ import 'express-validator' import 'multer' import * as validator from 'validator' -import { CONSTRAINTS_FIELDS, TORRENT_MIMETYPE_EXT, VIDEO_IMPORT_STATES } from '../../initializers' +import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers' import { exists, isFileValid } from './misc' import * as express from 'express' import { VideoImportModel } from '../../models/video/video-import' @@ -24,7 +24,7 @@ function isVideoImportStateValid (value: any) { return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined } -const videoTorrentImportTypes = Object.keys(TORRENT_MIMETYPE_EXT).map(m => `(${m})`) +const videoTorrentImportTypes = Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT).map(m => `(${m})`) const videoTorrentImportRegex = videoTorrentImportTypes.join('|') function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true) diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index a13b09ac8..e6f22e6c5 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -5,10 +5,9 @@ import 'multer' import * as validator from 'validator' import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' import { - CONSTRAINTS_FIELDS, + CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_CATEGORIES, VIDEO_LICENCES, - VIDEO_MIMETYPE_EXT, VIDEO_PRIVACIES, VIDEO_RATE_TYPES, VIDEO_STATES @@ -83,10 +82,15 @@ function isVideoRatingTypeValid (value: string) { return value === 'none' || values(VIDEO_RATE_TYPES).indexOf(value as VideoRateType) !== -1 } -const videoFileTypes = Object.keys(VIDEO_MIMETYPE_EXT).map(m => `(${m})`) -const videoFileTypesRegex = videoFileTypes.join('|') +function isVideoFileExtnameValid (value: string) { + return exists(value) && MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined +} function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { + const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) + .map(m => `(${m})`) + .join('|') + return isFileValid(files, videoFileTypesRegex, 'videofile', null) } @@ -221,6 +225,7 @@ export { isVideoStateValid, isVideoViewsValid, isVideoRatingTypeValid, + isVideoFileExtnameValid, isVideoDurationValid, isVideoTagValid, isVideoPrivacyValid, diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index b51c7cfba..a7bc7eec8 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -19,7 +19,7 @@ function checkMissedConfig () { 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'redundancy.videos.strategies', 'redundancy.videos.check_interval', - 'transcoding.enabled', 'transcoding.threads', + 'transcoding.enabled', 'transcoding.threads', 'transcoding.allow_additional_extensions', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'trending.videos.interval_days', 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index d4496bc34..ad61bee73 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -16,7 +16,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 290 +const LAST_MIGRATION_VERSION = 295 // --------------------------------------------------------------------------- @@ -246,6 +246,7 @@ const CONFIG = { }, TRANSCODING: { get ENABLED () { return config.get('transcoding.enabled') }, + get ALLOW_ADDITIONAL_EXTENSIONS () { return config.get('transcoding.allow_additional_extensions') }, get THREADS () { return config.get('transcoding.threads') }, RESOLUTIONS: { get '240p' () { return config.get('transcoding.resolutions.240p') }, @@ -298,7 +299,7 @@ const CONFIG = { // --------------------------------------------------------------------------- -const CONSTRAINTS_FIELDS = { +let CONSTRAINTS_FIELDS = { USERS: { NAME: { min: 1, max: 50 }, // Length DESCRIPTION: { min: 3, max: 1000 }, // Length @@ -357,7 +358,7 @@ const CONSTRAINTS_FIELDS = { max: 2 * 1024 * 1024 // 2MB } }, - EXTNAME: [ '.mp4', '.ogv', '.webm' ], + EXTNAME: buildVideosExtname(), INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 DURATION: { min: 0 }, // Number TAGS: { min: 0, max: 5 }, // Number of total tags @@ -480,27 +481,31 @@ const VIDEO_ABUSE_STATES = { [VideoAbuseState.ACCEPTED]: 'Accepted' } -const VIDEO_MIMETYPE_EXT = { - 'video/webm': '.webm', - 'video/ogg': '.ogv', - 'video/mp4': '.mp4' -} -const VIDEO_EXT_MIMETYPE = invert(VIDEO_MIMETYPE_EXT) - -const IMAGE_MIMETYPE_EXT = { - 'image/png': '.png', - 'image/jpg': '.jpg', - 'image/jpeg': '.jpg' -} - -const VIDEO_CAPTIONS_MIMETYPE_EXT = { - 'text/vtt': '.vtt', - 'application/x-subrip': '.srt' -} - -const TORRENT_MIMETYPE_EXT = { - 'application/x-bittorrent': '.torrent' +const MIMETYPES = { + VIDEO: { + MIMETYPE_EXT: buildVideoMimetypeExt(), + EXT_MIMETYPE: null as { [ id: string ]: string } + }, + IMAGE: { + MIMETYPE_EXT: { + 'image/png': '.png', + 'image/jpg': '.jpg', + 'image/jpeg': '.jpg' + } + }, + VIDEO_CAPTIONS: { + MIMETYPE_EXT: { + 'text/vtt': '.vtt', + 'application/x-subrip': '.srt' + } + }, + TORRENT: { + MIMETYPE_EXT: { + 'application/x-bittorrent': '.torrent' + } + } } +MIMETYPES.VIDEO.EXT_MIMETYPE = invert(MIMETYPES.VIDEO.MIMETYPE_EXT) // --------------------------------------------------------------------------- @@ -526,7 +531,7 @@ const ACTIVITY_PUB = { COLLECTION_ITEMS_PER_PAGE: 10, FETCH_PAGE_LIMIT: 100, URL_MIME_TYPES: { - VIDEO: Object.keys(VIDEO_MIMETYPE_EXT), + VIDEO: Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT), TORRENT: [ 'application/x-bittorrent' ], MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] }, @@ -685,13 +690,12 @@ if (isTestInstance() === true) { ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' } -updateWebserverConfig() +updateWebserverUrls() // --------------------------------------------------------------------------- export { API_VERSION, - VIDEO_CAPTIONS_MIMETYPE_EXT, AVATARS_SIZE, ACCEPT_HEADERS, BCRYPT_SALT_SIZE, @@ -719,7 +723,6 @@ export { FEEDS, JOB_TTL, NSFW_POLICY_TYPES, - TORRENT_MIMETYPE_EXT, STATIC_MAX_AGE, STATIC_PATHS, VIDEO_IMPORT_TIMEOUT, @@ -732,7 +735,6 @@ export { VIDEO_LICENCES, VIDEO_STATES, VIDEO_RATE_TYPES, - VIDEO_MIMETYPE_EXT, VIDEO_TRANSCODING_FPS, FFMPEG_NICE, VIDEO_ABUSE_STATES, @@ -740,13 +742,12 @@ export { USER_PASSWORD_RESET_LIFETIME, MEMOIZE_TTL, USER_EMAIL_VERIFY_LIFETIME, - IMAGE_MIMETYPE_EXT, OVERVIEWS, SCHEDULER_INTERVALS_MS, REPEAT_JOBS, STATIC_DOWNLOAD_PATHS, RATES_LIMIT, - VIDEO_EXT_MIMETYPE, + MIMETYPES, CRAWL_REQUEST_CONCURRENCY, JOB_COMPLETED_LIFETIME, HTTP_SIGNATURE, @@ -768,11 +769,43 @@ function getLocalConfigFilePath () { return join(dirname(configSources[ 0 ].name), filename + '.json') } -function updateWebserverConfig () { +function buildVideoMimetypeExt () { + const data = { + 'video/webm': '.webm', + 'video/ogg': '.ogv', + 'video/mp4': '.mp4' + } + + if (CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) { + Object.assign(data, { + 'video/quicktime': '.mov', + 'video/x-msvideo': '.avi', + 'video/x-flv': '.flv', + 'video/x-matroska': '.mkv' + }) + } + + return data +} + +function updateWebserverUrls () { CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) } +function updateWebserverConfig () { + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = buildVideosExtname() + + MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt() + MIMETYPES.VIDEO.EXT_MIMETYPE = invert(MIMETYPES.VIDEO.MIMETYPE_EXT) +} + +function buildVideosExtname () { + return CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS + ? [ '.mp4', '.ogv', '.webm', '.mkv', '.mov', '.avi', '.flv' ] + : [ '.mp4', '.ogv', '.webm' ] +} + function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { if (!objs) return [] @@ -854,4 +887,5 @@ export function reloadConfig () { config = require('config') updateWebserverConfig() + updateWebserverUrls() } diff --git a/server/initializers/migrations/0295-video-file-extname.ts b/server/initializers/migrations/0295-video-file-extname.ts new file mode 100644 index 000000000..dbf249f66 --- /dev/null +++ b/server/initializers/migrations/0295-video-file-extname.ts @@ -0,0 +1,49 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize, + db: any +}): Promise { + { + await utils.queryInterface.renameColumn('videoFile', 'extname', 'extname_old') + } + + { + const data = { + type: Sequelize.STRING, + defaultValue: null, + allowNull: true + } + + await utils.queryInterface.addColumn('videoFile', 'extname', data) + } + + { + const query = 'UPDATE "videoFile" SET "extname" = "extname_old"::text' + await utils.sequelize.query(query) + } + + { + const data = { + type: Sequelize.STRING, + defaultValue: null, + allowNull: false + } + await utils.queryInterface.changeColumn('videoFile', 'extname', data) + } + + { + await utils.queryInterface.removeColumn('videoFile', 'extname_old') + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index bbe48833d..f7bf7c65a 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -1,5 +1,4 @@ import * as Bluebird from 'bluebird' -import { join } from 'path' import { Transaction } from 'sequelize' import * as url from 'url' import * as uuidv4 from 'uuid/v4' @@ -13,7 +12,7 @@ import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' import { doRequest, downloadImage } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' +import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' @@ -172,10 +171,10 @@ async function fetchActorTotalItems (url: string) { async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { if ( - actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && + actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && isActivityPubUrlValid(actorJSON.icon.url) ) { - const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] + const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] const avatarName = uuidv4() + extension await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 3d17e6846..379c2a0d7 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -1,7 +1,6 @@ import * as Bluebird from 'bluebird' import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' -import { join } from 'path' import * as request from 'request' import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' @@ -11,7 +10,7 @@ import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { doRequest, downloadImage } from '../../helpers/requests' -import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_MIMETYPE_EXT } from '../../initializers' +import { ACTIVITY_PUB, CONFIG, MIMETYPES, REMOTE_SCHEME, sequelizeTypescript, THUMBNAILS_SIZE } from '../../initializers' import { ActorModel } from '../../models/activitypub/actor' import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' @@ -362,7 +361,7 @@ export { // --------------------------------------------------------------------------- function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { - const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) + const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) const urlMediaType = url.mediaType || url.mimeType return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') @@ -490,7 +489,7 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid const mediaType = fileUrl.mediaType || fileUrl.mimeType const attribute = { - extname: VIDEO_MIMETYPE_EXT[ mediaType ], + extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ], infoHash: parsed.infoHash, resolution: fileUrl.height, size: fileUrl.size, diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index dd37dad22..8b6cd146a 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -15,7 +15,7 @@ import { import { ActorModel } from '../activitypub/actor' import { getVideoSort, throwIfNotValid } from '../utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONFIG, CONSTRAINTS_FIELDS, STATIC_PATHS, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { CONFIG, CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers' import { VideoFileModel } from '../video/video-file' import { getServerActor } from '../../helpers/utils' import { VideoModel } from '../video/video' @@ -415,8 +415,8 @@ export class VideoRedundancyModel extends Model { expires: this.expiresOn.toISOString(), url: { type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, - mediaType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, + mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, href: this.fileUrl, height: this.VideoFile.resolution, size: this.VideoFile.size, diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index adebdf0c7..3fd2d5a99 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -14,6 +14,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { + isVideoFileExtnameValid, isVideoFileInfoHashValid, isVideoFileResolutionValid, isVideoFileSizeValid, @@ -58,7 +59,8 @@ export class VideoFileModel extends Model { size: number @AllowNull(false) - @Column(DataType.ENUM(values(CONSTRAINTS_FIELDS.VIDEOS.EXTNAME))) + @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname')) + @Column extname: string @AllowNull(false) diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index e3f8d525b..de0747f22 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -2,7 +2,7 @@ import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoModel } from './video' import { VideoFileModel } from './video-file' import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' -import { CONFIG, THUMBNAILS_SIZE, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' import { VideoCaptionModel } from './video-caption' import { getVideoCommentsActivityPubUrl, @@ -207,8 +207,8 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { for (const file of video.VideoFiles) { url.push({ type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, - mediaType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, + mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ file.extname ] as any, href: video.getVideoFileUrl(file, baseUrlHttp), height: file.resolution, size: file.size, diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index ffae380c1..b7bf41b58 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -54,6 +54,7 @@ describe('Test config API validators', function () { }, transcoding: { enabled: true, + allowAdditionalExtensions: true, threads: 1, resolutions: { '240p': false, diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index d94eccf8e..f26b91435 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -320,10 +320,15 @@ describe('Test videos API validator', function () { it('Should fail without an incorrect input file', async function () { const fields = baseCorrectParams - const attaches = { + let attaches = { 'videofile': join(__dirname, '..', '..', 'fixtures', 'video_short_fake.webm') } await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) + + attaches = { + 'videofile': join(__dirname, '..', '..', 'fixtures', 'video_short.mkv') + } + await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) }) it('Should fail with an incorrect thumbnail file', async function () { diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index c5c360a17..4c163d47d 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -17,6 +17,7 @@ import { setAccessTokensToServers, updateCustomConfig } from '../../../../shared/utils' +import { ServerConfig } from '../../../../shared/models' const expect = chai.expect @@ -43,6 +44,7 @@ function checkInitialConfig (data: CustomConfig) { expect(data.user.videoQuota).to.equal(5242880) expect(data.user.videoQuotaDaily).to.equal(-1) expect(data.transcoding.enabled).to.be.false + expect(data.transcoding.allowAdditionalExtensions).to.be.false expect(data.transcoding.threads).to.equal(2) expect(data.transcoding.resolutions['240p']).to.be.true expect(data.transcoding.resolutions['360p']).to.be.true @@ -74,6 +76,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.user.videoQuotaDaily).to.equal(318742) expect(data.transcoding.enabled).to.be.true expect(data.transcoding.threads).to.equal(1) + expect(data.transcoding.allowAdditionalExtensions).to.be.true expect(data.transcoding.resolutions['240p']).to.be.false expect(data.transcoding.resolutions['360p']).to.be.true expect(data.transcoding.resolutions['480p']).to.be.true @@ -96,7 +99,7 @@ describe('Test config', function () { it('Should have a correct config on a server with registration enabled', async function () { const res = await getConfig(server.url) - const data = res.body + const data: ServerConfig = res.body expect(data.signup.allowed).to.be.true }) @@ -111,11 +114,21 @@ describe('Test config', function () { ]) const res = await getConfig(server.url) - const data = res.body + const data: ServerConfig = res.body expect(data.signup.allowed).to.be.false }) + it('Should have the correct video allowed extensions', async function () { + const res = await getConfig(server.url) + const data: ServerConfig = res.body + + expect(data.video.file.extensions).to.have.lengthOf(3) + expect(data.video.file.extensions).to.contain('.mp4') + expect(data.video.file.extensions).to.contain('.webm') + expect(data.video.file.extensions).to.contain('.ogv') + }) + it('Should get the customized configuration', async function () { const res = await getCustomConfig(server.url, server.accessToken) const data = res.body as CustomConfig @@ -165,6 +178,7 @@ describe('Test config', function () { }, transcoding: { enabled: true, + allowAdditionalExtensions: true, threads: 1, resolutions: { '240p': false, @@ -193,6 +207,18 @@ describe('Test config', function () { checkUpdatedConfig(data) }) + it('Should have the correct updated video allowed extensions', async function () { + const res = await getConfig(server.url) + const data: ServerConfig = res.body + + expect(data.video.file.extensions).to.have.length.above(3) + expect(data.video.file.extensions).to.contain('.mp4') + expect(data.video.file.extensions).to.contain('.webm') + expect(data.video.file.extensions).to.contain('.ogv') + expect(data.video.file.extensions).to.contain('.flv') + expect(data.video.file.extensions).to.contain('.mkv') + }) + it('Should have the configuration updated after a restart', async function () { this.timeout(10000) diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 68cf00194..eefd32ef8 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -20,9 +20,8 @@ import { uploadVideo, webtorrentAdd } from '../../../../shared/utils' -import { join } from 'path' +import { extname, join } from 'path' import { waitJobs } from '../../../../shared/utils/server/jobs' -import { pathExists } from 'fs-extra' import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants' const expect = chai.expect @@ -322,6 +321,34 @@ describe('Test video transcoding', function () { } }) + it('Should accept and transcode additional extensions', async function () { + this.timeout(300000) + + for (const fixture of [ 'video_short.mkv', 'video_short.avi' ]) { + const videoAttributes = { + name: fixture, + fixture + } + + await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, videoAttributes) + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideosList(server.url) + + const video = res.body.data.find(v => v.name === videoAttributes.name) + const res2 = await getVideo(server.url, video.id) + const videoDetails = res2.body + + expect(videoDetails.files).to.have.lengthOf(4) + + const magnetUri = videoDetails.files[ 0 ].magnetUri + expect(magnetUri).to.contain('.mp4') + } + } + }) + after(async function () { killallServers(servers) }) diff --git a/server/tests/fixtures/video_short.avi b/server/tests/fixtures/video_short.avi new file mode 100644 index 000000000..88979cab2 Binary files /dev/null and b/server/tests/fixtures/video_short.avi differ diff --git a/server/tests/fixtures/video_short.mkv b/server/tests/fixtures/video_short.mkv new file mode 100644 index 000000000..a67f4f806 Binary files /dev/null and b/server/tests/fixtures/video_short.mkv differ -- cgit v1.2.3