10 files changed, 160 insertions, 56 deletions

diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 8bc3123bf..44f3d3ef7 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -10,6 +10,7 @@ import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '..
 import { objectConverter } from '../../helpers/core-utils'
 import { isSignupAllowed, isSignupAllowedForCurrentIP } from '../../helpers/signup'
 import { getServerCommit } from '../../helpers/utils'
+import { getEnabledResolutions } from '../../lib/video-transcoding'
 import { CONFIG, isEmailEnabled, reloadConfig } from '../../initializers/config'
 import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '../../initializers/constants'
 import { ClientHtml } from '../../lib/client-html'
@@ -285,16 +286,6 @@ function getRegisteredThemes () {
                       }))
 }
 
-function getEnabledResolutions (type: 'vod' | 'live') {
-  const transcoding = type === 'vod'
-    ? CONFIG.TRANSCODING
-    : CONFIG.LIVE.TRANSCODING
-
-  return Object.keys(transcoding.RESOLUTIONS)
-               .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true)
-               .map(r => parseInt(r, 10))
-}
-
 function getRegisteredPlugins () {
   return PluginManager.Instance.getRegisteredPlugins()
                       .map(p => ({
@@ -345,7 +336,6 @@ function getExternalAuthsPlugins () {
 
 export {
   configRouter,
-  getEnabledResolutions,
   getRegisteredPlugins,
   getRegisteredThemes
 }
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 82e084c54..9702e219a 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -146,9 +146,10 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
   } catch (err) {
     logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
 
-    return res.status(HttpStatusCode.BAD_REQUEST_400).json({
-      error: 'Cannot fetch remote information of this URL.'
-    }).end()
+    return res.status(HttpStatusCode.BAD_REQUEST_400)
+              .json({
+                error: 'Cannot fetch remote information of this URL.'
+              })
   }
 
   const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
@@ -219,9 +220,8 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
     videoImportId: videoImport.id,
     generateThumbnail: !thumbnailModel,
     generatePreview: !previewModel,
-    fileExt: youtubeDLInfo.fileExt
-      ? `.${youtubeDLInfo.fileExt}`
-      : '.mp4'
+    fileExt: `.${youtubeDLInfo.ext || 'mp4'}`,
+    mergeExt: youtubeDLInfo.mergeExt ? `.${youtubeDLInfo.mergeExt}` : ''
   }
   await JobQueue.Instance.createJobWithPromise({ type: 'video-import', payload })
 
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index bdb9c3041..a7b28704c 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -19,13 +19,14 @@ import { VideoCommentModel } from '../models/video/video-comment'
 import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo'
 import { join } from 'path'
 import { root } from '../helpers/core-utils'
+import { getEnabledResolutions } from '../lib/video-transcoding'
 import { CONFIG, isEmailEnabled } from '../initializers/config'
 import { getPreview, getVideoCaption } from './lazy-static'
 import { VideoStreamingPlaylistType } from '@shared/models/videos/video-streaming-playlist.type'
 import { MVideoFile, MVideoFullLight } from '@server/types/models'
 import { getTorrentFilePath, getVideoFilePath } from '@server/lib/video-paths'
 import { getThemeOrDefault } from '../lib/plugins/theme-utils'
-import { getEnabledResolutions, getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config'
+import { getRegisteredPlugins, getRegisteredThemes } from '@server/controllers/api/config'
 import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
 import { serveIndexHTML } from '@server/lib/client-html'
 
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 74e5f896c..ebb788e8e 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,6 +1,7 @@
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
+import { getEnabledResolutions } from '../lib/video-transcoding'
 import { join } from 'path'
 import { peertubeTruncate, root } from './core-utils'
 import { ensureDir, remove, writeFile } from 'fs-extra'
@@ -8,6 +9,7 @@ import * as request from 'request'
 import { createWriteStream } from 'fs'
 import { CONFIG } from '@server/initializers/config'
 import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
+import { VideoResolution } from '../../shared/models/videos'
 
 export type YoutubeDLInfo = {
   name?: string
@@ -18,7 +20,8 @@ export type YoutubeDLInfo = {
   nsfw?: boolean
   tags?: string[]
   thumbnailUrl?: string
-  fileExt?: string
+  ext?: string
+  mergeExt?: string
   originallyPublishedAt?: Date
 }
 
@@ -41,12 +44,21 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
     }
 
     args = wrapWithProxyOptions(args)
+    args = [ '-f', getYoutubeDLVideoFormat() ].concat(args)
 
     safeGetYoutubeDL()
       .then(youtubeDL => {
         youtubeDL.getInfo(url, args, processOptions, (err, info) => {
           if (err) return rej(err)
           if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
+          if (info.format_id?.includes('+')) {
+            // this is a merge format and its extension will be appended
+            if (info.ext === 'mp4') {
+              info.mergeExt = 'mp4'
+            } else {
+              info.mergeExt = 'mkv'
+            }
+          }
 
           const obj = buildVideoInfo(normalizeObject(info))
           if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
@@ -92,13 +104,40 @@ function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
   })
 }
 
-function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
+function getYoutubeDLVideoFormat () {
+  /**
+   * list of format selectors in order or preference
+   * see https://github.com/ytdl-org/youtube-dl#format-selection
+   *
+   * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
+   * of being able to do a "quick-transcode"
+   * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
+   * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
+   *
+   * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
+   **/
+  const enabledResolutions = getEnabledResolutions('vod')
+  const resolution = enabledResolutions.length === 0
+    ? VideoResolution.H_720P
+    : Math.max(...enabledResolutions)
+
+  return [
+    `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
+    `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
+    `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
+    `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
+    'best[vcodec!*=av01][vcodec!*=vp9.2]' // case fallback
+  ].join('/')
+}
+
+function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) {
   const path = generateVideoImportTmpPath(url, extension)
+  const finalPath = mergeExtension ? path.replace(new RegExp(`${extension}$`), mergeExtension) : path
   let timer
 
-  logger.info('Importing youtubeDL video %s to %s', url, path)
+  logger.info('Importing youtubeDL video %s to %s', url, finalPath)
 
-  let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
+  let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ]
   options = wrapWithProxyOptions(options)
 
   if (process.env.FFMPEG_PATH) {
@@ -118,7 +157,7 @@ function downloadYoutubeDLVideo (url: string, extension: string, timeout: number
             return rej(err)
           }
 
-          return res(path)
+          return res(finalPath)
         })
 
         timer = setTimeout(() => {
@@ -236,6 +275,7 @@ function buildOriginallyPublishedAt (obj: any) {
 
 export {
   updateYoutubeDLBinary,
+  getYoutubeDLVideoFormat,
   downloadYoutubeDLVideo,
   getYoutubeDLSubs,
   getYoutubeDLInfo,
@@ -275,7 +315,8 @@ function buildVideoInfo (obj: any): YoutubeDLInfo {
     tags: getTags(obj.tags),
     thumbnailUrl: obj.thumbnail || undefined,
     originallyPublishedAt: buildOriginallyPublishedAt(obj),
-    fileExt: obj.ext
+    ext: obj.ext,
+    mergeExt: obj.mergeExt
   }
 }
 
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 5a82a8d2b..db3112418 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -79,7 +79,11 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
     generatePreview: payload.generatePreview
   }
 
-  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options)
+  return processFile(
+    () => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT, payload.mergeExt),
+    videoImport,
+    options
+  )
 }
 
 async function getVideoImportOrDie (videoImportId: number) {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 078e85acf..a6b79eaea 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -201,6 +201,16 @@ function generateHlsPlaylist (options: {
   })
 }
 
+function getEnabledResolutions (type: 'vod' | 'live') {
+  const transcoding = type === 'vod'
+    ? CONFIG.TRANSCODING
+    : CONFIG.LIVE.TRANSCODING
+
+  return Object.keys(transcoding.RESOLUTIONS)
+               .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true)
+               .map(r => parseInt(r, 10))
+}
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -208,7 +218,8 @@ export {
   generateHlsPlaylistFromTS,
   optimizeOriginalVideofile,
   transcodeNewResolution,
-  mergeAudioVideofile
+  mergeAudioVideofile,
+  getEnabledResolutions
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts
index 0d41933a6..c53af3861 100644
--- a/server/middlewares/validators/videos/video-imports.ts
+++ b/server/middlewares/validators/videos/video-imports.ts
@@ -43,7 +43,7 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
 
     if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
 
-    if (req.body.targetUrl && CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
+    if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true && req.body.targetUrl) {
       cleanUpReqFiles(req)
       return res.status(HttpStatusCode.CONFLICT_409)
         .json({ error: 'HTTP import is not enabled on this instance.' })
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index 8d19a4274..61e7a81ee 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -14,12 +14,19 @@ import {
   listVideoCaptions,
   ServerInfo,
   setAccessTokensToServers,
-  testCaptionFile
+  testCaptionFile,
+  updateCustomSubConfig
 } from '../../../../shared/extra-utils'
 import { areHttpImportTestsDisabled, testImage } from '../../../../shared/extra-utils/miscs/miscs'
 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
-import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
-import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos'
+import {
+  getMagnetURI,
+  getMyVideoImports,
+  getYoutubeHDRVideoUrl,
+  getYoutubeVideoUrl,
+  importVideo
+} from '../../../../shared/extra-utils/videos/video-imports'
+import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy, VideoResolution } from '../../../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -90,7 +97,7 @@ describe('Test video imports', function () {
   }
 
   before(async function () {
-    this.timeout(30000)
+    this.timeout(30_000)
 
     // Run servers
     servers = await flushAndRunMultipleServers(2)
@@ -111,7 +118,7 @@ describe('Test video imports', function () {
   })
 
   it('Should import videos on server 1', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const baseAttributes = {
       channelId: channelIdServer1,
@@ -223,7 +230,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should have the video listed on the two instances', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     await waitJobs(servers)
 
@@ -238,7 +245,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should import a video on server 2 with some fields', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const attributes = {
       targetUrl: getYoutubeVideoUrl(),
@@ -256,7 +263,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should have the videos listed on the two instances', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     await waitJobs(servers)
 
@@ -273,7 +280,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should import a video that will be transcoded', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     const attributes = {
       name: 'transcoded video',
@@ -295,6 +302,56 @@ Ajouter un sous-titre est vraiment facile`)
     }
   })
 
+  it('Should import no HDR version on a HDR video', async function () {
+    this.timeout(120_000)
+
+    const config = {
+      transcoding: {
+        enabled: true,
+        resolutions: {
+          '240p': false,
+          '360p': false,
+          '480p': false,
+          '720p': false,
+          '1080p': true, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
+          '1440p': false,
+          '2160p': false
+        },
+        webtorrent: { enabled: true },
+        hls: { enabled: false }
+      },
+      import: {
+        videos: {
+          http: {
+            enabled: true
+          },
+          torrent: {
+            enabled: true
+          }
+        }
+      }
+    }
+    await updateCustomSubConfig(servers[0].url, servers[0].accessToken, config)
+
+    const attributes = {
+      name: 'hdr video',
+      targetUrl: getYoutubeHDRVideoUrl(),
+      channelId: channelIdServer1,
+      privacy: VideoPrivacy.PUBLIC
+    }
+    const res1 = await importVideo(servers[0].url, servers[0].accessToken, attributes)
+    const videoUUID = res1.body.video.uuid
+
+    await waitJobs(servers)
+
+    // test resolution
+    const res2 = await getVideo(servers[0].url, videoUUID)
+    const video: VideoDetails = res2.body
+    expect(video.name).to.equal('hdr video')
+    const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
+    expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_1080P)
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index 817d9faf2..32f566506 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -45,7 +45,7 @@ describe('Test video transcoding', function () {
   let servers: ServerInfo[] = []
 
   before(async function () {
-    this.timeout(30000)
+    this.timeout(30_000)
 
     // Run servers
     servers = await flushAndRunMultipleServers(2)
@@ -56,7 +56,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should not transcode video on server 1', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'my super name for server 1',
@@ -86,7 +86,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode video on server 2', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     const videoAttributes = {
       name: 'my super name for server 2',
@@ -117,7 +117,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode high bit rate mp3 to proper bit rate', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'mp3_256k',
@@ -149,7 +149,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode video with no audio and have no audio itself', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'no_audio',
@@ -174,7 +174,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should leave the audio untouched, but properly transcode the video', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'untouched_audio',
@@ -209,7 +209,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode a 60 FPS video', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'my super 30fps name for server 2',
@@ -248,7 +248,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should wait for transcoding before publishing the video', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     {
       // Upload the video, but wait transcoding
@@ -301,7 +301,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should respect maximum bitrate values', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     let tempFixturePath: string
 
@@ -341,7 +341,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should accept and transcode additional extensions', async function () {
-    this.timeout(300000)
+    this.timeout(300_000)
 
     let tempFixturePath: string
 
@@ -378,14 +378,14 @@ describe('Test video transcoding', function () {
   })
 
   it('Should correctly detect if quick transcode is possible', async function () {
-    this.timeout(10000)
+    this.timeout(10_000)
 
     expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.mp4'))).to.be.true
     expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.webm'))).to.be.false
   })
 
   it('Should merge an audio file with the preview file', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
     await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
@@ -410,7 +410,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should upload an audio file and choose a default background image', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
     await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
@@ -435,7 +435,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should downscale to the closest divisor standard framerate', async function () {
-    this.timeout(200000)
+    this.timeout(200_000)
 
     let tempFixturePath: string
 
@@ -476,7 +476,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should not transcode to an higher bitrate than the original file', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     const config = {
       transcoding: {
@@ -508,12 +508,12 @@ describe('Test video transcoding', function () {
 
     const resolutions = [ 240, 360, 480, 720, 1080 ]
     for (const r of resolutions) {
-      expect(await getServerFileSize(servers[1], `videos/${videoUUID}-${r}.mp4`)).to.be.below(60000)
+      expect(await getServerFileSize(servers[1], `videos/${videoUUID}-${r}.mp4`)).to.be.below(60_000)
     }
   })
 
   it('Should provide valid ffprobe data', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     const videoUUID = (await uploadVideoAndGetId({ server: servers[1], videoName: 'ffprobe data' })).uuid
     await waitJobs(servers)
@@ -570,7 +570,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode a 4k video', async function () {
-    this.timeout(200000)
+    this.timeout(200_000)
 
     const videoAttributes = {
       name: '4k video',
diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
index 5fc5a867c..3a82b3832 100644
--- a/server/tools/peertube-import-videos.ts
+++ b/server/tools/peertube-import-videos.ts
@@ -11,7 +11,7 @@ import * as prompt from 'prompt'
 import { accessSync, constants } from 'fs'
 import { remove } from 'fs-extra'
 import { sha256 } from '../helpers/core-utils'
-import { buildOriginallyPublishedAt, safeGetYoutubeDL } from '../helpers/youtube-dl'
+import { buildOriginallyPublishedAt, getYoutubeDLVideoFormat, safeGetYoutubeDL } from '../helpers/youtube-dl'
 import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli'
 
 type UserInfo = {
@@ -156,7 +156,7 @@ function processVideo (parameters: {
 
     log.info('Downloading video "%s"...', videoInfo.title)
 
-    const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', ...command.args, '-o', path ]
+    const options = [ '-f', getYoutubeDLVideoFormat(), ...command.args, '-o', path ]
     try {
       const youtubeDL = await safeGetYoutubeDL()
       youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {