aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-08-10 14:25:29 +0200
committerChocobozzz <chocobozzz@cpy.re>2020-08-11 07:57:20 +0200
commit30bc55c88b3b7416c2224925e88639694fd32746 (patch)
treeaebe3b1b8657198e432080f7d15a5694f19a8389 /server
parent560605026bfadab711cf6d34e9c7ea865887816a (diff)
downloadPeerTube-30bc55c88b3b7416c2224925e88639694fd32746.tar.gz
PeerTube-30bc55c88b3b7416c2224925e88639694fd32746.tar.zst
PeerTube-30bc55c88b3b7416c2224925e88639694fd32746.zip
Refactor video extensions logic in server
Diffstat (limited to 'server')
-rw-r--r--server/helpers/custom-validators/videos.ts6
-rw-r--r--server/helpers/express-utils.ts9
-rw-r--r--server/helpers/video.ts19
-rw-r--r--server/initializers/constants.ts41
-rw-r--r--server/lib/activitypub/videos.ts60
-rw-r--r--server/tests/api/server/config.ts4
6 files changed, 87 insertions, 52 deletions
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 60e8075f6..40fecc09b 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -81,11 +81,7 @@ function isVideoFileExtnameValid (value: string) {
81} 81}
82 82
83function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { 83function isVideoFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
84 const videoFileTypesRegex = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 84 return isFileValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile', null)
85 .map(m => `(${m})`)
86 .join('|')
87
88 return isFileValid(files, videoFileTypesRegex, 'videofile', null)
89} 85}
90 86
91const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME 87const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index f46812977..ba23557ba 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -6,6 +6,7 @@ import { deleteFileAsync, generateRandomString } from './utils'
6import { extname } from 'path' 6import { extname } from 'path'
7import { isArray } from './custom-validators/misc' 7import { isArray } from './custom-validators/misc'
8import { CONFIG } from '../initializers/config' 8import { CONFIG } from '../initializers/config'
9import { getExtFromMimetype } from './video'
9 10
10function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { 11function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
11 if (paramNSFW === 'true') return true 12 if (paramNSFW === 'true') return true
@@ -65,7 +66,7 @@ function badRequest (req: express.Request, res: express.Response) {
65 66
66function createReqFiles ( 67function createReqFiles (
67 fieldNames: string[], 68 fieldNames: string[],
68 mimeTypes: { [id: string]: string }, 69 mimeTypes: { [id: string]: string | string[] },
69 destinations: { [fieldName: string]: string } 70 destinations: { [fieldName: string]: string }
70) { 71) {
71 const storage = multer.diskStorage({ 72 const storage = multer.diskStorage({
@@ -76,13 +77,13 @@ function createReqFiles (
76 filename: async (req, file, cb) => { 77 filename: async (req, file, cb) => {
77 let extension: string 78 let extension: string
78 const fileExtension = extname(file.originalname) 79 const fileExtension = extname(file.originalname)
79 const extensionFromMimetype = mimeTypes[file.mimetype] 80 const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
80 81
81 // Take the file extension if we don't understand the mime type 82 // Take the file extension if we don't understand the mime type
82 // We have the OGG/OGV exception too because firefox sends a bad mime type when sending an OGG file 83 if (!extensionFromMimetype) {
83 if (fileExtension === '.ogg' || fileExtension === '.ogv' || !extensionFromMimetype) {
84 extension = fileExtension 84 extension = fileExtension
85 } else { 85 } else {
86 // Take the first available extension for this mimetype
86 extension = extensionFromMimetype 87 extension = extensionFromMimetype
87 } 88 }
88 89
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 89c85accb..488b4da17 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,5 +1,8 @@
1import { VideoModel } from '../models/video/video'
2import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { Response } from 'express'
3import { CONFIG } from '@server/initializers/config'
4import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
5import { JobQueue } from '@server/lib/job-queue'
3import { 6import {
4 isStreamingPlaylist, 7 isStreamingPlaylist,
5 MStreamingPlaylistVideo, 8 MStreamingPlaylistVideo,
@@ -12,11 +15,8 @@ import {
12 MVideoThumbnail, 15 MVideoThumbnail,
13 MVideoWithRights 16 MVideoWithRights
14} from '@server/types/models' 17} from '@server/types/models'
15import { Response } from 'express'
16import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
17import { JobQueue } from '@server/lib/job-queue'
18import { VideoPrivacy, VideoTranscodingPayload } from '@shared/models' 18import { VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
19import { CONFIG } from "@server/initializers/config" 19import { VideoModel } from '../models/video/video'
20 20
21type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes' 21type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'
22 22
@@ -110,6 +110,14 @@ function getPrivaciesForFederation () {
110 : [ { privacy: VideoPrivacy.PUBLIC } ] 110 : [ { privacy: VideoPrivacy.PUBLIC } ]
111} 111}
112 112
113function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) {
114 const value = mimeTypes[mimeType]
115
116 if (Array.isArray(value)) return value[0]
117
118 return value
119}
120
113export { 121export {
114 VideoFetchType, 122 VideoFetchType,
115 VideoFetchByUrlType, 123 VideoFetchByUrlType,
@@ -118,6 +126,7 @@ export {
118 fetchVideoByUrl, 126 fetchVideoByUrl,
119 addOptimizeOrMergeAudioJob, 127 addOptimizeOrMergeAudioJob,
120 extractVideo, 128 extractVideo,
129 getExtFromMimetype,
121 isPrivacyForFederation, 130 isPrivacyForFederation,
122 getPrivaciesForFederation 131 getPrivaciesForFederation
123} 132}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 573d86b60..ebbdba262 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -422,7 +422,8 @@ const MIMETYPES = {
422 EXT_MIMETYPE: null as { [ id: string ]: string } 422 EXT_MIMETYPE: null as { [ id: string ]: string }
423 }, 423 },
424 VIDEO: { 424 VIDEO: {
425 MIMETYPE_EXT: null as { [ id: string ]: string }, 425 MIMETYPE_EXT: null as { [ id: string ]: string | string[] },
426 MIMETYPES_REGEX: null as string,
426 EXT_MIMETYPE: null as { [ id: string ]: string } 427 EXT_MIMETYPE: null as { [ id: string ]: string }
427 }, 428 },
428 IMAGE: { 429 IMAGE: {
@@ -825,15 +826,19 @@ function buildVideoMimetypeExt () {
825 const data = { 826 const data = {
826 // streamable formats that warrant cross-browser compatibility 827 // streamable formats that warrant cross-browser compatibility
827 'video/webm': '.webm', 828 'video/webm': '.webm',
828 'video/ogg': '.ogv', 829 // We'll add .ogg if additional extensions are enabled
830 // We could add .ogg here but since it could be an audio file,
831 // it would be confusing for users because PeerTube will refuse their file (based on the mimetype)
832 'video/ogg': [ '.ogv' ],
829 'video/mp4': '.mp4' 833 'video/mp4': '.mp4'
830 } 834 }
831 835
832 if (CONFIG.TRANSCODING.ENABLED) { 836 if (CONFIG.TRANSCODING.ENABLED) {
833 if (CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) { 837 if (CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) {
838 data['video/ogg'].push('.ogg')
839
834 Object.assign(data, { 840 Object.assign(data, {
835 'video/x-matroska': '.mkv', 841 'video/x-matroska': '.mkv',
836 'video/ogg': '.ogg',
837 842
838 // Developed by Apple 843 // Developed by Apple
839 'video/quicktime': '.mov', // often used as output format by editing software 844 'video/quicktime': '.mov', // often used as output format by editing software
@@ -892,14 +897,36 @@ function updateWebserverUrls () {
892 897
893function updateWebserverConfig () { 898function updateWebserverConfig () {
894 MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt() 899 MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt()
895 MIMETYPES.VIDEO.EXT_MIMETYPE = invert(MIMETYPES.VIDEO.MIMETYPE_EXT) 900 MIMETYPES.VIDEO.MIMETYPES_REGEX = buildMimetypesRegex(MIMETYPES.VIDEO.MIMETYPE_EXT)
901
896 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 902 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
897 903
898 CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = buildVideosExtname() 904 MIMETYPES.VIDEO.EXT_MIMETYPE = buildVideoExtMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT)
905
906 CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = Object.keys(MIMETYPES.VIDEO.EXT_MIMETYPE)
907}
908
909function buildVideoExtMimetype (obj: { [ id: string ]: string | string[] }) {
910 const result: { [id: string]: string } = {}
911
912 for (const mimetype of Object.keys(obj)) {
913 const value = obj[mimetype]
914 if (!value) continue
915
916 const extensions = Array.isArray(value) ? value : [ value ]
917
918 for (const extension of extensions) {
919 result[extension] = mimetype
920 }
921 }
922
923 return result
899} 924}
900 925
901function buildVideosExtname () { 926function buildMimetypesRegex (obj: { [id: string]: string | string[] }) {
902 return Object.keys(MIMETYPES.VIDEO.EXT_MIMETYPE).filter(e => e !== 'null') 927 return Object.keys(obj)
928 .map(m => `(${m})`)
929 .join('|')
903} 930}
904 931
905function loadLanguages () { 932function loadLanguages () {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 6c5f7f306..cbbf23be1 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -1,12 +1,15 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import * as sequelize from 'sequelize' 2import { maxBy, minBy } from 'lodash'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import { join } from 'path'
4import * as request from 'request' 5import * as request from 'request'
6import * as sequelize from 'sequelize'
5import { 7import {
6 ActivityHashTagObject, 8 ActivityHashTagObject,
7 ActivityMagnetUrlObject, 9 ActivityMagnetUrlObject,
8 ActivityPlaylistSegmentHashesObject, 10 ActivityPlaylistSegmentHashesObject,
9 ActivityPlaylistUrlObject, ActivitypubHttpFetcherPayload, 11 ActivityPlaylistUrlObject,
12 ActivitypubHttpFetcherPayload,
10 ActivityTagObject, 13 ActivityTagObject,
11 ActivityUrlObject, 14 ActivityUrlObject,
12 ActivityVideoUrlObject, 15 ActivityVideoUrlObject,
@@ -14,11 +17,16 @@ import {
14} from '../../../shared/index' 17} from '../../../shared/index'
15import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 18import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
16import { VideoPrivacy } from '../../../shared/models/videos' 19import { VideoPrivacy } from '../../../shared/models/videos'
20import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
21import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
22import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
17import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 23import { isAPVideoFileMetadataObject, sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
24import { isArray } from '../../helpers/custom-validators/misc'
18import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 25import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
19import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 26import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
20import { logger } from '../../helpers/logger' 27import { logger } from '../../helpers/logger'
21import { doRequest } from '../../helpers/requests' 28import { doRequest } from '../../helpers/requests'
29import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
22import { 30import {
23 ACTIVITY_PUB, 31 ACTIVITY_PUB,
24 MIMETYPES, 32 MIMETYPES,
@@ -28,33 +36,15 @@ import {
28 STATIC_PATHS, 36 STATIC_PATHS,
29 THUMBNAILS_SIZE 37 THUMBNAILS_SIZE
30} from '../../initializers/constants' 38} from '../../initializers/constants'
39import { sequelizeTypescript } from '../../initializers/database'
40import { AccountVideoRateModel } from '../../models/account/account-video-rate'
31import { TagModel } from '../../models/video/tag' 41import { TagModel } from '../../models/video/tag'
32import { VideoModel } from '../../models/video/video' 42import { VideoModel } from '../../models/video/video'
33import { VideoFileModel } from '../../models/video/video-file'
34import { getOrCreateActorAndServerAndModel } from './actor'
35import { addVideoComments } from './video-comments'
36import { crawlCollectionPage } from './crawl'
37import { sendCreateVideo, sendUpdateVideo } from './send'
38import { isArray } from '../../helpers/custom-validators/misc'
39import { VideoCaptionModel } from '../../models/video/video-caption' 43import { VideoCaptionModel } from '../../models/video/video-caption'
40import { JobQueue } from '../job-queue'
41import { createRates } from './video-rates'
42import { addVideoShares, shareVideoByServerAndChannel } from './share'
43import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
44import { buildRemoteVideoBaseUrl, checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
45import { Notifier } from '../notifier'
46import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
47import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
48import { AccountVideoRateModel } from '../../models/account/account-video-rate'
49import { VideoShareModel } from '../../models/video/video-share'
50import { VideoCommentModel } from '../../models/video/video-comment' 44import { VideoCommentModel } from '../../models/video/video-comment'
51import { sequelizeTypescript } from '../../initializers/database' 45import { VideoFileModel } from '../../models/video/video-file'
52import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail' 46import { VideoShareModel } from '../../models/video/video-share'
53import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' 47import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
54import { join } from 'path'
55import { FilteredModelAttributes } from '../../types/sequelize'
56import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
57import { ActorFollowScoreCache } from '../files-cache'
58import { 48import {
59 MAccountIdActor, 49 MAccountIdActor,
60 MChannelAccountLight, 50 MChannelAccountLight,
@@ -73,7 +63,18 @@ import {
73 MVideoThumbnail 63 MVideoThumbnail
74} from '../../types/models' 64} from '../../types/models'
75import { MThumbnail } from '../../types/models/video/thumbnail' 65import { MThumbnail } from '../../types/models/video/thumbnail'
76import { maxBy, minBy } from 'lodash' 66import { FilteredModelAttributes } from '../../types/sequelize'
67import { ActorFollowScoreCache } from '../files-cache'
68import { JobQueue } from '../job-queue'
69import { Notifier } from '../notifier'
70import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail'
71import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
72import { getOrCreateActorAndServerAndModel } from './actor'
73import { crawlCollectionPage } from './crawl'
74import { sendCreateVideo, sendUpdateVideo } from './send'
75import { addVideoShares, shareVideoByServerAndChannel } from './share'
76import { addVideoComments } from './video-comments'
77import { createRates } from './video-rates'
77 78
78async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) { 79async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: sequelize.Transaction) {
79 const video = videoArg as MVideoAP 80 const video = videoArg as MVideoAP
@@ -516,10 +517,9 @@ export {
516// --------------------------------------------------------------------------- 517// ---------------------------------------------------------------------------
517 518
518function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { 519function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
519 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
520
521 const urlMediaType = url.mediaType 520 const urlMediaType = url.mediaType
522 return mimeTypes.includes(urlMediaType) && urlMediaType.startsWith('video/') 521
522 return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/')
523} 523}
524 524
525function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { 525function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
@@ -716,7 +716,7 @@ function videoFileActivityUrlToDBAttributes (
716 716
717 const mediaType = fileUrl.mediaType 717 const mediaType = fileUrl.mediaType
718 const attribute = { 718 const attribute = {
719 extname: MIMETYPES.VIDEO.MIMETYPE_EXT[mediaType], 719 extname: getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, mediaType),
720 infoHash: parsed.infoHash, 720 infoHash: parsed.infoHash,
721 resolution: fileUrl.height, 721 resolution: fileUrl.height,
722 size: fileUrl.size, 722 size: fileUrl.size,
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index f5183042c..60efd332c 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -363,10 +363,12 @@ describe('Test config', function () {
363 }) 363 })
364 364
365 it('Should have the correct updated video allowed extensions', async function () { 365 it('Should have the correct updated video allowed extensions', async function () {
366 this.timeout(10000)
367
366 const res = await getConfig(server.url) 368 const res = await getConfig(server.url)
367 const data: ServerConfig = res.body 369 const data: ServerConfig = res.body
368 370
369 expect(data.video.file.extensions).to.have.length.above(3) 371 expect(data.video.file.extensions).to.have.length.above(4)
370 expect(data.video.file.extensions).to.contain('.mp4') 372 expect(data.video.file.extensions).to.contain('.mp4')
371 expect(data.video.file.extensions).to.contain('.webm') 373 expect(data.video.file.extensions).to.contain('.webm')
372 expect(data.video.file.extensions).to.contain('.ogv') 374 expect(data.video.file.extensions).to.contain('.ogv')