aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorRigel Kent <sendmemail@rigelk.eu>2021-01-15 15:56:56 +0100
committerGitHub <noreply@github.com>2021-01-15 15:56:56 +0100
commit454c20fa7cdb05eba7f1be3c83389b54807af0b3 (patch)
treee9b364da69eaf88f0470414c506c3b5631984c60
parentd43c6b1ffc5e6c895f9e9f9de6625f17a9755c20 (diff)
downloadPeerTube-454c20fa7cdb05eba7f1be3c83389b54807af0b3.tar.gz
PeerTube-454c20fa7cdb05eba7f1be3c83389b54807af0b3.tar.zst
PeerTube-454c20fa7cdb05eba7f1be3c83389b54807af0b3.zip
stricter youtubedl format selectors (#3516)
* stricter youtubedl format selectors make sure selectors avoid av1, and otherwise match as closely to the maximum resolution enabled for transcoding * add support for merge formats in youtubedl * avoid vp9.2 in youtubedl to avoid any HDR * move getEnabledResolutions, safer replace of imported extension * add test for youtube-dl selectors
-rw-r--r--client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html2
-rw-r--r--client/src/app/+my-library/my-video-imports/my-video-imports.component.html6
-rw-r--r--server/controllers/api/config.ts12
-rw-r--r--server/controllers/api/videos/import.ts12
-rw-r--r--server/controllers/static.ts3
-rw-r--r--server/helpers/youtube-dl.ts53
-rw-r--r--server/lib/job-queue/handlers/video-import.ts6
-rw-r--r--server/lib/video-transcoding.ts13
-rw-r--r--server/middlewares/validators/videos/video-imports.ts2
-rw-r--r--server/tests/api/videos/video-imports.ts75
-rw-r--r--server/tests/api/videos/video-transcoder.ts36
-rw-r--r--server/tools/peertube-import-videos.ts4
-rw-r--r--shared/core-utils/miscs/http-error-codes.ts8
-rw-r--r--shared/extra-utils/videos/video-imports.ts24
-rw-r--r--shared/models/server/job.model.ts1
15 files changed, 196 insertions, 61 deletions
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
index 72b7ceb73..09e7e96ac 100644
--- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
+++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
@@ -753,7 +753,7 @@
753 i18n-labelText labelText="Allow additional extensions" 753 i18n-labelText labelText="Allow additional extensions"
754 > 754 >
755 <ng-container ngProjectAs="description"> 755 <ng-container ngProjectAs="description">
756 <span i18n>Allows users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, m2ts, .mxf, or .nut videos.</span> 756 <span i18n>Allows users to upload .mkv, .mov, .avi, .wmv, .flv, .f4v, .3g2, .3gp, .mts, .m2ts, .mxf, or .nut videos.</span>
757 </ng-container> 757 </ng-container>
758 </my-peertube-checkbox> 758 </my-peertube-checkbox>
759 </div> 759 </div>
diff --git a/client/src/app/+my-library/my-video-imports/my-video-imports.component.html b/client/src/app/+my-library/my-video-imports/my-video-imports.component.html
index 1d3a45f76..9ae85c0ca 100644
--- a/client/src/app/+my-library/my-video-imports/my-video-imports.component.html
+++ b/client/src/app/+my-library/my-video-imports/my-video-imports.component.html
@@ -23,8 +23,8 @@
23 23
24 <ng-template pTemplate="body" let-expanded="expanded" let-videoImport> 24 <ng-template pTemplate="body" let-expanded="expanded" let-videoImport>
25 <tr> 25 <tr>
26 <td class="expand-cell"> 26 <td class="expand-cell c-hand" [pRowToggler]="videoImport" i18n-ngbTooltip ngbTooltip="See the error" placement="top-left" container="body">
27 <span *ngIf="videoImport.error" class="expander" [pRowToggler]="videoImport" i18n-ngbTooltip ngbTooltip="See the error"> 27 <span *ngIf="videoImport.error" class="expander">
28 <i [ngClass]="expanded ? 'glyphicon glyphicon-menu-down' : 'glyphicon glyphicon-menu-right'"></i> 28 <i [ngClass]="expanded ? 'glyphicon glyphicon-menu-down' : 'glyphicon glyphicon-menu-right'"></i>
29 </span> 29 </span>
30 </td> 30 </td>
@@ -51,7 +51,7 @@
51 </td> 51 </td>
52 52
53 <td> 53 <td>
54 <span class="badge" [ngClass]="getVideoImportStateClass(videoImport.state)"> 54 <span class="badge" [ngClass]="getVideoImportStateClass(videoImport.state.id)">
55 {{ videoImport.state.label }} 55 {{ videoImport.state.label }}
56 </span> 56 </span>
57 </td> 57 </td>
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 8bc3123bf..44f3d3ef7 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -10,6 +10,7 @@ import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '..
10import { objectConverter } from '../../helpers/core-utils' 10import { objectConverter } from '../../helpers/core-utils'
11import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup' 11import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup'
12import { getServerCommit } from '../../helpers/utils' 12import { getServerCommit } from '../../helpers/utils'
13import { getEnabledResolutions } from '../../lib/video-transcoding'
13import { CONFIG, isEmailEnabled, reloadConfig } from '../../initializers/config' 14import { CONFIG, isEmailEnabled, reloadConfig } from '../../initializers/config'
14import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '../../initializers/constants' 15import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '../../initializers/constants'
15import { ClientHtml } from '../../lib/client-html' 16import { ClientHtml } from '../../lib/client-html'
@@ -285,16 +286,6 @@ function getRegisteredThemes () {
285 })) 286 }))
286} 287}
287 288
288function getEnabledResolutions (type: 'vod' | 'live') {
289 const transcoding = type === 'vod'
290 ? CONFIG.TRANSCODING
291 : CONFIG.LIVE.TRANSCODING
292
293 return Object.keys(transcoding.RESOLUTIONS)
294 .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true)
295 .map(r => parseInt(r, 10))
296}
297
298function getRegisteredPlugins () { 289function getRegisteredPlugins () {
299 return PluginManager.Instance.getRegisteredPlugins() 290 return PluginManager.Instance.getRegisteredPlugins()
300 .map(p => ({ 291 .map(p => ({
@@ -345,7 +336,6 @@ function getExternalAuthsPlugins () {
345 336
346export { 337export {
347 configRouter, 338 configRouter,
348 getEnabledResolutions,
349 getRegisteredPlugins, 339 getRegisteredPlugins,
350 getRegisteredThemes 340 getRegisteredThemes
351} 341}
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 82e084c54..9702e219a 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -146,9 +146,10 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
146 } catch (err) { 146 } catch (err) {
147 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) 147 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
148 148
149 return res.status(HttpStatusCode.BAD_REQUEST_400).json({ 149 return res.status(HttpStatusCode.BAD_REQUEST_400)
150 error: 'Cannot fetch remote information of this URL.' 150 .json({
151 }).end() 151 error: 'Cannot fetch remote information of this URL.'
152 })
152 } 153 }
153 154
154 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) 155 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
@@ -219,9 +220,8 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
219 videoImportId: videoImport.id, 220 videoImportId: videoImport.id,
220 generateThumbnail: !thumbnailModel, 221 generateThumbnail: !thumbnailModel,
221 generatePreview: !previewModel, 222 generatePreview: !previewModel,
222 fileExt: youtubeDLInfo.fileExt 223 fileExt: `.${youtubeDLInfo.ext || 'mp4'}`,
223 ? `.${youtubeDLInfo.fileExt}` 224 mergeExt: youtubeDLInfo.mergeExt ? `.${youtubeDLInfo.mergeExt}` : ''
224 : '.mp4'
225 } 225 }
226 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload }) 226 await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
227 227
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index bdb9c3041..a7b28704c 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -19,13 +19,14 @@ import { VideoCommentModel } from '../models/video/video-comment'
19import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo' 19import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
20import { join } from 'path' 20import { join } from 'path'
21import { root } from '../helpers/core-utils' 21import { root } from '../helpers/core-utils'
22import { getEnabledResolutions } from '../lib/video-transcoding'
22import { CONFIG, isEmailEnabled } from '../initializers/config' 23import { CONFIG, isEmailEnabled } from '../initializers/config'
23import { getPreview, getVideoCaption } from './lazy-static' 24import { getPreview, getVideoCaption } from './lazy-static'
24import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type' 25import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type'
25import { MVideoFile, MVideoFullLight } from '@server/types/models' 26import { MVideoFile, MVideoFullLight } from '@server/types/models'
26import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths' 27import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths'
27import { getThemeOrDefault } from '../lib/plugins/theme-utils' 28import { getThemeOrDefault } from '../lib/plugins/theme-utils'
28import { getEnabledResolutions, getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config' 29import { getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config'
29import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes' 30import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
30import { serveIndexHTML } from '@server/lib/client-html' 31import { serveIndexHTML } from '@server/lib/client-html'
31 32
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 74e5f896c..ebb788e8e 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,6 +1,7 @@
1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' 1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
2import { logger } from './logger' 2import { logger } from './logger'
3import { generateVideoImportTmpPath } from './utils' 3import { generateVideoImportTmpPath } from './utils'
4import { getEnabledResolutions } from '../lib/video-transcoding'
4import { join } from 'path' 5import { join } from 'path'
5import { peertubeTruncate, root } from './core-utils' 6import { peertubeTruncate, root } from './core-utils'
6import { ensureDir, remove, writeFile } from 'fs-extra' 7import { ensureDir, remove, writeFile } from 'fs-extra'
@@ -8,6 +9,7 @@ import * as request from 'request'
8import { createWriteStream } from 'fs' 9import { createWriteStream } from 'fs'
9import { CONFIG } from '@server/initializers/config' 10import { CONFIG } from '@server/initializers/config'
10import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' 11import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
12import { VideoResolution } from '../../shared/models/videos'
11 13
12export type YoutubeDLInfo = { 14export type YoutubeDLInfo = {
13 name?: string 15 name?: string
@@ -18,7 +20,8 @@ export type YoutubeDLInfo = {
18 nsfw?: boolean 20 nsfw?: boolean
19 tags?: string[] 21 tags?: string[]
20 thumbnailUrl?: string 22 thumbnailUrl?: string
21 fileExt?: string 23 ext?: string
24 mergeExt?: string
22 originallyPublishedAt?: Date 25 originallyPublishedAt?: Date
23} 26}
24 27
@@ -41,12 +44,21 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
41 } 44 }
42 45
43 args = wrapWithProxyOptions(args) 46 args = wrapWithProxyOptions(args)
47 args = [ '-f', getYoutubeDLVideoFormat() ].concat(args)
44 48
45 safeGetYoutubeDL() 49 safeGetYoutubeDL()
46 .then(youtubeDL => { 50 .then(youtubeDL => {
47 youtubeDL.getInfo(url, args, processOptions, (err, info) => { 51 youtubeDL.getInfo(url, args, processOptions, (err, info) => {
48 if (err) return rej(err) 52 if (err) return rej(err)
49 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) 53 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
54 if (info.format_id?.includes('+')) {
55 // this is a merge format and its extension will be appended
56 if (info.ext === 'mp4') {
57 info.mergeExt = 'mp4'
58 } else {
59 info.mergeExt = 'mkv'
60 }
61 }
50 62
51 const obj = buildVideoInfo(normalizeObject(info)) 63 const obj = buildVideoInfo(normalizeObject(info))
52 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' 64 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
@@ -92,13 +104,40 @@ function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
92 }) 104 })
93} 105}
94 106
95function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) { 107function getYoutubeDLVideoFormat () {
108 /**
109 * list of format selectors in order or preference
110 * see https://github.com/ytdl-org/youtube-dl#format-selection
111 *
112 * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
113 * of being able to do a "quick-transcode"
114 * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
115 * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
116 *
117 * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
118 **/
119 const enabledResolutions = getEnabledResolutions('vod')
120 const resolution = enabledResolutions.length === 0
121 ? VideoResolution.H_720P
122 : Math.max(...enabledResolutions)
123
124 return [
125 `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
126 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
127 `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
128 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
129 'best[vcodec!*=av01][vcodec!*=vp9.2]' // case fallback
130 ].join('/')
131}
132
133function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) {
96 const path = generateVideoImportTmpPath(url, extension) 134 const path = generateVideoImportTmpPath(url, extension)
135 const finalPath = mergeExtension ? path.replace(new RegExp(`${extension}$`), mergeExtension) : path
97 let timer 136 let timer
98 137
99 logger.info('Importing youtubeDL video %s to %s', url, path) 138 logger.info('Importing youtubeDL video %s to %s', url, finalPath)
100 139
101 let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] 140 let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ]
102 options = wrapWithProxyOptions(options) 141 options = wrapWithProxyOptions(options)
103 142
104 if (process.env.FFMPEG_PATH) { 143 if (process.env.FFMPEG_PATH) {
@@ -118,7 +157,7 @@ function downloadYoutubeDLVideo (url: string, extension: string, timeout: number
118 return rej(err) 157 return rej(err)
119 } 158 }
120 159
121 return res(path) 160 return res(finalPath)
122 }) 161 })
123 162
124 timer = setTimeout(() => { 163 timer = setTimeout(() => {
@@ -236,6 +275,7 @@ function buildOriginallyPublishedAt (obj: any) {
236 275
237export { 276export {
238 updateYoutubeDLBinary, 277 updateYoutubeDLBinary,
278 getYoutubeDLVideoFormat,
239 downloadYoutubeDLVideo, 279 downloadYoutubeDLVideo,
240 getYoutubeDLSubs, 280 getYoutubeDLSubs,
241 getYoutubeDLInfo, 281 getYoutubeDLInfo,
@@ -275,7 +315,8 @@ function buildVideoInfo (obj: any): YoutubeDLInfo {
275 tags: getTags(obj.tags), 315 tags: getTags(obj.tags),
276 thumbnailUrl: obj.thumbnail || undefined, 316 thumbnailUrl: obj.thumbnail || undefined,
277 originallyPublishedAt: buildOriginallyPublishedAt(obj), 317 originallyPublishedAt: buildOriginallyPublishedAt(obj),
278 fileExt: obj.ext 318 ext: obj.ext,
319 mergeExt: obj.mergeExt
279 } 320 }
280} 321}
281 322
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 5a82a8d2b..db3112418 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -79,7 +79,11 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
79 generatePreview: payload.generatePreview 79 generatePreview: payload.generatePreview
80 } 80 }
81 81
82 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options) 82 return processFile(
83 () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT, payload.mergeExt),
84 videoImport,
85 options
86 )
83} 87}
84 88
85async function getVideoImportOrDie (videoImportId: number) { 89async function getVideoImportOrDie (videoImportId: number) {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 078e85acf..a6b79eaea 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -201,6 +201,16 @@ function generateHlsPlaylist (options: {
201 }) 201 })
202} 202}
203 203
204function getEnabledResolutions (type: 'vod' | 'live') {
205 const transcoding = type === 'vod'
206 ? CONFIG.TRANSCODING
207 : CONFIG.LIVE.TRANSCODING
208
209 return Object.keys(transcoding.RESOLUTIONS)
210 .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true)
211 .map(r => parseInt(r, 10))
212}
213
204// --------------------------------------------------------------------------- 214// ---------------------------------------------------------------------------
205 215
206export { 216export {
@@ -208,7 +218,8 @@ export {
208 generateHlsPlaylistFromTS, 218 generateHlsPlaylistFromTS,
209 optimizeOriginalVideofile, 219 optimizeOriginalVideofile,
210 transcodeNewResolution, 220 transcodeNewResolution,
211 mergeAudioVideofile 221 mergeAudioVideofile,
222 getEnabledResolutions
212} 223}
213 224
214// --------------------------------------------------------------------------- 225// ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts
index 0d41933a6..c53af3861 100644
--- a/server/middlewares/validators/videos/video-imports.ts
+++ b/server/middlewares/validators/videos/video-imports.ts
@@ -43,7 +43,7 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
43 43
44 if (areValidationErrors(req, res)) return cleanUpReqFiles(req) 44 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
45 45
46 if (req.body.targetUrl && CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) { 46 if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true && req.body.targetUrl) {
47 cleanUpReqFiles(req) 47 cleanUpReqFiles(req)
48 return res.status(HttpStatusCode.CONFLICT_409) 48 return res.status(HttpStatusCode.CONFLICT_409)
49 .json({ error: 'HTTP import is not enabled on this instance.' }) 49 .json({ error: 'HTTP import is not enabled on this instance.' })
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index 8d19a4274..61e7a81ee 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -14,12 +14,19 @@ import {
14 listVideoCaptions, 14 listVideoCaptions,
15 ServerInfo, 15 ServerInfo,
16 setAccessTokensToServers, 16 setAccessTokensToServers,
17 testCaptionFile 17 testCaptionFile,
18 updateCustomSubConfig
18} from '../../../../shared/extra-utils' 19} from '../../../../shared/extra-utils'
19import { areHttpImportTestsDisabled, testImage } from '../../../../shared/extra-utils/miscs/miscs' 20import { areHttpImportTestsDisabled, testImage } from '../../../../shared/extra-utils/miscs/miscs'
20import { waitJobs } from '../../../../shared/extra-utils/server/jobs' 21import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
21import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' 22import {
22import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' 23 getMagnetURI,
24 getMyVideoImports,
25 getYoutubeHDRVideoUrl,
26 getYoutubeVideoUrl,
27 importVideo
28} from '../../../../shared/extra-utils/videos/video-imports'
29import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy, VideoResolution } from '../../../../shared/models/videos'
23 30
24const expect = chai.expect 31const expect = chai.expect
25 32
@@ -90,7 +97,7 @@ describe('Test video imports', function () {
90 } 97 }
91 98
92 before(async function () { 99 before(async function () {
93 this.timeout(30000) 100 this.timeout(30_000)
94 101
95 // Run servers 102 // Run servers
96 servers = await flushAndRunMultipleServers(2) 103 servers = await flushAndRunMultipleServers(2)
@@ -111,7 +118,7 @@ describe('Test video imports', function () {
111 }) 118 })
112 119
113 it('Should import videos on server 1', async function () { 120 it('Should import videos on server 1', async function () {
114 this.timeout(60000) 121 this.timeout(60_000)
115 122
116 const baseAttributes = { 123 const baseAttributes = {
117 channelId: channelIdServer1, 124 channelId: channelIdServer1,
@@ -223,7 +230,7 @@ Ajouter un sous-titre est vraiment facile`)
223 }) 230 })
224 231
225 it('Should have the video listed on the two instances', async function () { 232 it('Should have the video listed on the two instances', async function () {
226 this.timeout(120000) 233 this.timeout(120_000)
227 234
228 await waitJobs(servers) 235 await waitJobs(servers)
229 236
@@ -238,7 +245,7 @@ Ajouter un sous-titre est vraiment facile`)
238 }) 245 })
239 246
240 it('Should import a video on server 2 with some fields', async function () { 247 it('Should import a video on server 2 with some fields', async function () {
241 this.timeout(60000) 248 this.timeout(60_000)
242 249
243 const attributes = { 250 const attributes = {
244 targetUrl: getYoutubeVideoUrl(), 251 targetUrl: getYoutubeVideoUrl(),
@@ -256,7 +263,7 @@ Ajouter un sous-titre est vraiment facile`)
256 }) 263 })
257 264
258 it('Should have the videos listed on the two instances', async function () { 265 it('Should have the videos listed on the two instances', async function () {
259 this.timeout(120000) 266 this.timeout(120_000)
260 267
261 await waitJobs(servers) 268 await waitJobs(servers)
262 269
@@ -273,7 +280,7 @@ Ajouter un sous-titre est vraiment facile`)
273 }) 280 })
274 281
275 it('Should import a video that will be transcoded', async function () { 282 it('Should import a video that will be transcoded', async function () {
276 this.timeout(120000) 283 this.timeout(120_000)
277 284
278 const attributes = { 285 const attributes = {
279 name: 'transcoded video', 286 name: 'transcoded video',
@@ -295,6 +302,56 @@ Ajouter un sous-titre est vraiment facile`)
295 } 302 }
296 }) 303 })
297 304
305 it('Should import no HDR version on a HDR video', async function () {
306 this.timeout(120_000)
307
308 const config = {
309 transcoding: {
310 enabled: true,
311 resolutions: {
312 '240p': false,
313 '360p': false,
314 '480p': false,
315 '720p': false,
316 '1080p': true, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
317 '1440p': false,
318 '2160p': false
319 },
320 webtorrent: { enabled: true },
321 hls: { enabled: false }
322 },
323 import: {
324 videos: {
325 http: {
326 enabled: true
327 },
328 torrent: {
329 enabled: true
330 }
331 }
332 }
333 }
334 await updateCustomSubConfig(servers[0].url, servers[0].accessToken, config)
335
336 const attributes = {
337 name: 'hdr video',
338 targetUrl: getYoutubeHDRVideoUrl(),
339 channelId: channelIdServer1,
340 privacy: VideoPrivacy.PUBLIC
341 }
342 const res1 = await importVideo(servers[0].url, servers[0].accessToken, attributes)
343 const videoUUID = res1.body.video.uuid
344
345 await waitJobs(servers)
346
347 // test resolution
348 const res2 = await getVideo(servers[0].url, videoUUID)
349 const video: VideoDetails = res2.body
350 expect(video.name).to.equal('hdr video')
351 const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
352 expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_1080P)
353 })
354
298 after(async function () { 355 after(async function () {
299 await cleanupTests(servers) 356 await cleanupTests(servers)
300 }) 357 })
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index 817d9faf2..32f566506 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -45,7 +45,7 @@ describe('Test video transcoding', function () {
45 let servers: ServerInfo[] = [] 45 let servers: ServerInfo[] = []
46 46
47 before(async function () { 47 before(async function () {
48 this.timeout(30000) 48 this.timeout(30_000)
49 49
50 // Run servers 50 // Run servers
51 servers = await flushAndRunMultipleServers(2) 51 servers = await flushAndRunMultipleServers(2)
@@ -56,7 +56,7 @@ describe('Test video transcoding', function () {
56 }) 56 })
57 57
58 it('Should not transcode video on server 1', async function () { 58 it('Should not transcode video on server 1', async function () {
59 this.timeout(60000) 59 this.timeout(60_000)
60 60
61 const videoAttributes = { 61 const videoAttributes = {
62 name: 'my super name for server 1', 62 name: 'my super name for server 1',
@@ -86,7 +86,7 @@ describe('Test video transcoding', function () {
86 }) 86 })
87 87
88 it('Should transcode video on server 2', async function () { 88 it('Should transcode video on server 2', async function () {
89 this.timeout(120000) 89 this.timeout(120_000)
90 90
91 const videoAttributes = { 91 const videoAttributes = {
92 name: 'my super name for server 2', 92 name: 'my super name for server 2',
@@ -117,7 +117,7 @@ describe('Test video transcoding', function () {
117 }) 117 })
118 118
119 it('Should transcode high bit rate mp3 to proper bit rate', async function () { 119 it('Should transcode high bit rate mp3 to proper bit rate', async function () {
120 this.timeout(60000) 120 this.timeout(60_000)
121 121
122 const videoAttributes = { 122 const videoAttributes = {
123 name: 'mp3_256k', 123 name: 'mp3_256k',
@@ -149,7 +149,7 @@ describe('Test video transcoding', function () {
149 }) 149 })
150 150
151 it('Should transcode video with no audio and have no audio itself', async function () { 151 it('Should transcode video with no audio and have no audio itself', async function () {
152 this.timeout(60000) 152 this.timeout(60_000)
153 153
154 const videoAttributes = { 154 const videoAttributes = {
155 name: 'no_audio', 155 name: 'no_audio',
@@ -174,7 +174,7 @@ describe('Test video transcoding', function () {
174 }) 174 })
175 175
176 it('Should leave the audio untouched, but properly transcode the video', async function () { 176 it('Should leave the audio untouched, but properly transcode the video', async function () {
177 this.timeout(60000) 177 this.timeout(60_000)
178 178
179 const videoAttributes = { 179 const videoAttributes = {
180 name: 'untouched_audio', 180 name: 'untouched_audio',
@@ -209,7 +209,7 @@ describe('Test video transcoding', function () {
209 }) 209 })
210 210
211 it('Should transcode a 60 FPS video', async function () { 211 it('Should transcode a 60 FPS video', async function () {
212 this.timeout(60000) 212 this.timeout(60_000)
213 213
214 const videoAttributes = { 214 const videoAttributes = {
215 name: 'my super 30fps name for server 2', 215 name: 'my super 30fps name for server 2',
@@ -248,7 +248,7 @@ describe('Test video transcoding', function () {
248 }) 248 })
249 249
250 it('Should wait for transcoding before publishing the video', async function () { 250 it('Should wait for transcoding before publishing the video', async function () {
251 this.timeout(160000) 251 this.timeout(160_000)
252 252
253 { 253 {
254 // Upload the video, but wait transcoding 254 // Upload the video, but wait transcoding
@@ -301,7 +301,7 @@ describe('Test video transcoding', function () {
301 }) 301 })
302 302
303 it('Should respect maximum bitrate values', async function () { 303 it('Should respect maximum bitrate values', async function () {
304 this.timeout(160000) 304 this.timeout(160_000)
305 305
306 let tempFixturePath: string 306 let tempFixturePath: string
307 307
@@ -341,7 +341,7 @@ describe('Test video transcoding', function () {
341 }) 341 })
342 342
343 it('Should accept and transcode additional extensions', async function () { 343 it('Should accept and transcode additional extensions', async function () {
344 this.timeout(300000) 344 this.timeout(300_000)
345 345
346 let tempFixturePath: string 346 let tempFixturePath: string
347 347
@@ -378,14 +378,14 @@ describe('Test video transcoding', function () {
378 }) 378 })
379 379
380 it('Should correctly detect if quick transcode is possible', async function () { 380 it('Should correctly detect if quick transcode is possible', async function () {
381 this.timeout(10000) 381 this.timeout(10_000)
382 382
383 expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.mp4'))).to.be.true 383 expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.mp4'))).to.be.true
384 expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.webm'))).to.be.false 384 expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.webm'))).to.be.false
385 }) 385 })
386 386
387 it('Should merge an audio file with the preview file', async function () { 387 it('Should merge an audio file with the preview file', async function () {
388 this.timeout(60000) 388 this.timeout(60_000)
389 389
390 const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } 390 const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
391 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) 391 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
@@ -410,7 +410,7 @@ describe('Test video transcoding', function () {
410 }) 410 })
411 411
412 it('Should upload an audio file and choose a default background image', async function () { 412 it('Should upload an audio file and choose a default background image', async function () {
413 this.timeout(60000) 413 this.timeout(60_000)
414 414
415 const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } 415 const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
416 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) 416 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
@@ -435,7 +435,7 @@ describe('Test video transcoding', function () {
435 }) 435 })
436 436
437 it('Should downscale to the closest divisor standard framerate', async function () { 437 it('Should downscale to the closest divisor standard framerate', async function () {
438 this.timeout(200000) 438 this.timeout(200_000)
439 439
440 let tempFixturePath: string 440 let tempFixturePath: string
441 441
@@ -476,7 +476,7 @@ describe('Test video transcoding', function () {
476 }) 476 })
477 477
478 it('Should not transcode to an higher bitrate than the original file', async function () { 478 it('Should not transcode to an higher bitrate than the original file', async function () {
479 this.timeout(160000) 479 this.timeout(160_000)
480 480
481 const config = { 481 const config = {
482 transcoding: { 482 transcoding: {
@@ -508,12 +508,12 @@ describe('Test video transcoding', function () {
508 508
509 const resolutions = [ 240, 360, 480, 720, 1080 ] 509 const resolutions = [ 240, 360, 480, 720, 1080 ]
510 for (const r of resolutions) { 510 for (const r of resolutions) {
511 expect(await getServerFileSize(servers[1], `videos/${videoUUID}-${r}.mp4`)).to.be.below(60000) 511 expect(await getServerFileSize(servers[1], `videos/${videoUUID}-${r}.mp4`)).to.be.below(60_000)
512 } 512 }
513 }) 513 })
514 514
515 it('Should provide valid ffprobe data', async function () { 515 it('Should provide valid ffprobe data', async function () {
516 this.timeout(160000) 516 this.timeout(160_000)
517 517
518 const videoUUID = (await uploadVideoAndGetId({ server: servers[1], videoName: 'ffprobe data' })).uuid 518 const videoUUID = (await uploadVideoAndGetId({ server: servers[1], videoName: 'ffprobe data' })).uuid
519 await waitJobs(servers) 519 await waitJobs(servers)
@@ -570,7 +570,7 @@ describe('Test video transcoding', function () {
570 }) 570 })
571 571
572 it('Should transcode a 4k video', async function () { 572 it('Should transcode a 4k video', async function () {
573 this.timeout(200000) 573 this.timeout(200_000)
574 574
575 const videoAttributes = { 575 const videoAttributes = {
576 name: '4k video', 576 name: '4k video',
diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
index 5fc5a867c..3a82b3832 100644
--- a/server/tools/peertube-import-videos.ts
+++ b/server/tools/peertube-import-videos.ts
@@ -11,7 +11,7 @@ import * as prompt from 'prompt'
11import { accessSync, constants } from 'fs' 11import { accessSync, constants } from 'fs'
12import { remove } from 'fs-extra' 12import { remove } from 'fs-extra'
13import { sha256 } from '../helpers/core-utils' 13import { sha256 } from '../helpers/core-utils'
14import { buildOriginallyPublishedAt, safeGetYoutubeDL } from '../helpers/youtube-dl' 14import { buildOriginallyPublishedAt, getYoutubeDLVideoFormat, safeGetYoutubeDL } from '../helpers/youtube-dl'
15import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli' 15import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli'
16 16
17type UserInfo = { 17type UserInfo = {
@@ -156,7 +156,7 @@ function processVideo (parameters: {
156 156
157 log.info('Downloading video "%s"...', videoInfo.title) 157 log.info('Downloading video "%s"...', videoInfo.title)
158 158
159 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', ...command.args, '-o', path ] 159 const options = [ '-f', getYoutubeDLVideoFormat(), ...command.args, '-o', path ]
160 try { 160 try {
161 const youtubeDL = await safeGetYoutubeDL() 161 const youtubeDL = await safeGetYoutubeDL()
162 youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => { 162 youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
diff --git a/shared/core-utils/miscs/http-error-codes.ts b/shared/core-utils/miscs/http-error-codes.ts
index 9ac8a6c83..b2fbdfc5a 100644
--- a/shared/core-utils/miscs/http-error-codes.ts
+++ b/shared/core-utils/miscs/http-error-codes.ts
@@ -196,6 +196,8 @@ export enum HttpStatusCode {
196 * 196 *
197 * Indicates that the request could not be processed because of conflict in the request, 197 * Indicates that the request could not be processed because of conflict in the request,
198 * such as an edit conflict between multiple simultaneous updates. 198 * such as an edit conflict between multiple simultaneous updates.
199 *
200 * @see HttpStatusCode.UNPROCESSABLE_ENTITY_422 to denote a disabled feature
199 */ 201 */
200 CONFLICT_409 = 409, 202 CONFLICT_409 = 409,
201 203
@@ -269,6 +271,12 @@ export enum HttpStatusCode {
269 * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.3 271 * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.3
270 * 272 *
271 * The request was well-formed but was unable to be followed due to semantic errors. 273 * The request was well-formed but was unable to be followed due to semantic errors.
274 * The server understands the content type of the request entity (hence a 415 (Unsupported Media Type) status code is inappropriate),
275 * and the syntax of the request entity is correct (thus a 400 (Bad Request) status code is inappropriate) but was unable to process
276 * the contained instructions. For example, this error condition may occur if an JSON request body contains well-formed (i.e.,
277 * syntactically correct), but semantically erroneous, JSON instructions.
278 *
279 * Can also be used to denote disabled features (akin to disabled syntax).
272 * 280 *
273 * @see HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 if the `Content-Type` was not supported. 281 * @see HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 if the `Content-Type` was not supported.
274 * @see HttpStatusCode.BAD_REQUEST_400 if the request was not parsable (broken JSON, XML) 282 * @see HttpStatusCode.BAD_REQUEST_400 if the request was not parsable (broken JSON, XML)
diff --git a/shared/extra-utils/videos/video-imports.ts b/shared/extra-utils/videos/video-imports.ts
index 52e0075fb..259b8a314 100644
--- a/shared/extra-utils/videos/video-imports.ts
+++ b/shared/extra-utils/videos/video-imports.ts
@@ -4,7 +4,28 @@ import { makeGetRequest, makeUploadRequest } from '../requests/requests'
4import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' 4import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
5 5
6function getYoutubeVideoUrl () { 6function getYoutubeVideoUrl () {
7 return 'http://www.youtube.com/watch?v=msX3jv1XdvM' 7 return 'https://www.youtube.com/watch?v=msX3jv1XdvM'
8}
9
10function getYoutubeHDRVideoUrl () {
11 /**
12 * The video is used to check format-selection correctness wrt. HDR,
13 * which brings its own set of oddities outside of a MediaSource.
14 * FIXME: refactor once HDR is supported at playback
15 *
16 * The video needs to have the following format_ids:
17 * (which you can check by using `youtube-dl <url> -F`):
18 * - 303 (1080p webm vp9)
19 * - 299 (1080p mp4 avc1)
20 * - 335 (1080p webm vp9.2 HDR)
21 *
22 * 15 jan. 2021: TEST VIDEO NOT CURRENTLY PROVIDING
23 * - 400 (1080p mp4 av01)
24 * - 315 (2160p webm vp9 HDR)
25 * - 337 (2160p webm vp9.2 HDR)
26 * - 401 (2160p mp4 av01 HDR)
27 */
28 return 'https://www.youtube.com/watch?v=MSJ25EqI19c'
8} 29}
9 30
10function getMagnetURI () { 31function getMagnetURI () {
@@ -61,6 +82,7 @@ function getMyVideoImports (url: string, token: string, sort?: string) {
61export { 82export {
62 getBadVideoUrl, 83 getBadVideoUrl,
63 getYoutubeVideoUrl, 84 getYoutubeVideoUrl,
85 getYoutubeHDRVideoUrl,
64 importVideo, 86 importVideo,
65 getMagnetURI, 87 getMagnetURI,
66 getMyVideoImports, 88 getMyVideoImports,
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index b0ed860a7..2af2a25a6 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -82,6 +82,7 @@ export type VideoImportYoutubeDLPayload = {
82 generatePreview: boolean 82 generatePreview: boolean
83 83
84 fileExt?: string 84 fileExt?: string
85 mergeExt?: string
85} 86}
86export type VideoImportTorrentPayload = { 87export type VideoImportTorrentPayload = {
87 type: VideoImportTorrentPayloadType 88 type: VideoImportTorrentPayloadType