stricter youtubedl format selectors (#3516)

* stricter youtubedl format selectors

make sure selectors avoid av1, and otherwise match as closely to the
maximum resolution enabled for transcoding

* add support for merge formats in youtubedl

* avoid vp9.2 in youtubedl to avoid any HDR

* move getEnabledResolutions, safer replace of imported extension

* add test for youtube-dl selectors

2 files changed, 84 insertions, 27 deletions

diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index 8d19a4274..61e7a81ee 100644
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -14,12 +14,19 @@ import {
   listVideoCaptions,
   ServerInfo,
   setAccessTokensToServers,
-  testCaptionFile
+  testCaptionFile,
+  updateCustomSubConfig
 } from '../../../../shared/extra-utils'
 import { areHttpImportTestsDisabled, testImage } from '../../../../shared/extra-utils/miscs/miscs'
 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
-import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
-import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos'
+import {
+  getMagnetURI,
+  getMyVideoImports,
+  getYoutubeHDRVideoUrl,
+  getYoutubeVideoUrl,
+  importVideo
+} from '../../../../shared/extra-utils/videos/video-imports'
+import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy, VideoResolution } from '../../../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -90,7 +97,7 @@ describe('Test video imports', function () {
   }
 
   before(async function () {
-    this.timeout(30000)
+    this.timeout(30_000)
 
     // Run servers
     servers = await flushAndRunMultipleServers(2)
@@ -111,7 +118,7 @@ describe('Test video imports', function () {
   })
 
   it('Should import videos on server 1', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const baseAttributes = {
       channelId: channelIdServer1,
@@ -223,7 +230,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should have the video listed on the two instances', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     await waitJobs(servers)
 
@@ -238,7 +245,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should import a video on server 2 with some fields', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const attributes = {
       targetUrl: getYoutubeVideoUrl(),
@@ -256,7 +263,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should have the videos listed on the two instances', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     await waitJobs(servers)
 
@@ -273,7 +280,7 @@ Ajouter un sous-titre est vraiment facile`)
   })
 
   it('Should import a video that will be transcoded', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     const attributes = {
       name: 'transcoded video',
@@ -295,6 +302,56 @@ Ajouter un sous-titre est vraiment facile`)
     }
   })
 
+  it('Should import no HDR version on a HDR video', async function () {
+    this.timeout(120_000)
+
+    const config = {
+      transcoding: {
+        enabled: true,
+        resolutions: {
+          '240p': false,
+          '360p': false,
+          '480p': false,
+          '720p': false,
+          '1080p': true, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
+          '1440p': false,
+          '2160p': false
+        },
+        webtorrent: { enabled: true },
+        hls: { enabled: false }
+      },
+      import: {
+        videos: {
+          http: {
+            enabled: true
+          },
+          torrent: {
+            enabled: true
+          }
+        }
+      }
+    }
+    await updateCustomSubConfig(servers[0].url, servers[0].accessToken, config)
+
+    const attributes = {
+      name: 'hdr video',
+      targetUrl: getYoutubeHDRVideoUrl(),
+      channelId: channelIdServer1,
+      privacy: VideoPrivacy.PUBLIC
+    }
+    const res1 = await importVideo(servers[0].url, servers[0].accessToken, attributes)
+    const videoUUID = res1.body.video.uuid
+
+    await waitJobs(servers)
+
+    // test resolution
+    const res2 = await getVideo(servers[0].url, videoUUID)
+    const video: VideoDetails = res2.body
+    expect(video.name).to.equal('hdr video')
+    const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
+    expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_1080P)
+  })
+
   after(async function () {
     await cleanupTests(servers)
   })
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index 817d9faf2..32f566506 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -45,7 +45,7 @@ describe('Test video transcoding', function () {
   let servers: ServerInfo[] = []
 
   before(async function () {
-    this.timeout(30000)
+    this.timeout(30_000)
 
     // Run servers
     servers = await flushAndRunMultipleServers(2)
@@ -56,7 +56,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should not transcode video on server 1', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'my super name for server 1',
@@ -86,7 +86,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode video on server 2', async function () {
-    this.timeout(120000)
+    this.timeout(120_000)
 
     const videoAttributes = {
       name: 'my super name for server 2',
@@ -117,7 +117,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode high bit rate mp3 to proper bit rate', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'mp3_256k',
@@ -149,7 +149,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode video with no audio and have no audio itself', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'no_audio',
@@ -174,7 +174,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should leave the audio untouched, but properly transcode the video', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'untouched_audio',
@@ -209,7 +209,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode a 60 FPS video', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributes = {
       name: 'my super 30fps name for server 2',
@@ -248,7 +248,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should wait for transcoding before publishing the video', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     {
       // Upload the video, but wait transcoding
@@ -301,7 +301,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should respect maximum bitrate values', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     let tempFixturePath: string
 
@@ -341,7 +341,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should accept and transcode additional extensions', async function () {
-    this.timeout(300000)
+    this.timeout(300_000)
 
     let tempFixturePath: string
 
@@ -378,14 +378,14 @@ describe('Test video transcoding', function () {
   })
 
   it('Should correctly detect if quick transcode is possible', async function () {
-    this.timeout(10000)
+    this.timeout(10_000)
 
     expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.mp4'))).to.be.true
     expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.webm'))).to.be.false
   })
 
   it('Should merge an audio file with the preview file', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
     await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
@@ -410,7 +410,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should upload an audio file and choose a default background image', async function () {
-    this.timeout(60000)
+    this.timeout(60_000)
 
     const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
     await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
@@ -435,7 +435,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should downscale to the closest divisor standard framerate', async function () {
-    this.timeout(200000)
+    this.timeout(200_000)
 
     let tempFixturePath: string
 
@@ -476,7 +476,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should not transcode to an higher bitrate than the original file', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     const config = {
       transcoding: {
@@ -508,12 +508,12 @@ describe('Test video transcoding', function () {
 
     const resolutions = [ 240, 360, 480, 720, 1080 ]
     for (const r of resolutions) {
-      expect(await getServerFileSize(servers[1], `videos/${videoUUID}-${r}.mp4`)).to.be.below(60000)
+      expect(await getServerFileSize(servers[1], `videos/${videoUUID}-${r}.mp4`)).to.be.below(60_000)
     }
   })
 
   it('Should provide valid ffprobe data', async function () {
-    this.timeout(160000)
+    this.timeout(160_000)
 
     const videoUUID = (await uploadVideoAndGetId({ server: servers[1], videoName: 'ffprobe data' })).uuid
     await waitJobs(servers)
@@ -570,7 +570,7 @@ describe('Test video transcoding', function () {
   })
 
   it('Should transcode a 4k video', async function () {
-    this.timeout(200000)
+    this.timeout(200_000)
 
     const videoAttributes = {
       name: '4k video',