Limit import depending on transcoding resolutions

diff --git a/scripts/create-transcoding-job.ts b/scripts/create-transcoding-job.ts
index b7761597efa19bc4f2035f54b873488d09d62619..f8c0ed4618e4149457ae88c8a2541d4411fce497 100755 (executable)
--- a/scripts/create-transcoding-job.ts
+++ b/scripts/create-transcoding-job.ts
@@ -53,7 +53,7 @@ async function run () {
   if (options.generateHls || CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
     const resolutionsEnabled = options.resolution
       ? [ parseInt(options.resolution) ]
-      : computeResolutionsToTranscode({ inputResolution: maxResolution, type: 'vod', includeInputResolution: true })
+      : computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false })
 
     for (const resolution of resolutionsEnabled) {
       dataInput.push({

diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index ff2fa9d864a566f18bb98b1b5e2a70e8dff2ab71..19bd2888cf8fe1f6518b18814a9697014cc606cc 100644 (file)
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -10,6 +10,7 @@ import { CONFIG, reloadConfig } from '../../initializers/config'
 import { ClientHtml } from '../../lib/client-html'
 import { asyncMiddleware, authenticate, ensureUserHasRight, openapiOperationDoc } from '../../middlewares'
 import { customConfigUpdateValidator, ensureConfigIsEditable } from '../../middlewares/validators/config'
+import { logger } from '@server/helpers/logger'
 
 const configRouter = express.Router()
 
@@ -112,6 +113,7 @@ async function updateCustomConfig (req: express.Request, res: express.Response)
 
   const data = customConfig()
 
+  logger.info('coucou', { data })
   auditLogger.update(
     getAuditIdFromRes(res),
     new CustomConfigAuditView(data),

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 7576e77e73808bc052ea82811dfecbabfee18937..b1295363003a6c7e33b1cf04cc88c6e0a76d7aa4 100644 (file)
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -175,7 +175,11 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
   const targetUrl = body.targetUrl
   const user = res.locals.oauth.token.User
 
-  const youtubeDL = new YoutubeDLWrapper(targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
+  const youtubeDL = new YoutubeDLWrapper(
+    targetUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
 
   // Get video infos
   let youtubeDLInfo: YoutubeDLInfo

diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts
index 7d924c5ac4b4f863d891bcfd0ea107bc9a294bec..b2b71a870aea243c452a96e93c83f628af9fcdec 100644 (file)
--- a/server/controllers/api/videos/transcoding.ts
+++ b/server/controllers/api/videos/transcoding.ts
@@ -32,7 +32,7 @@ async function createTranscoding (req: express.Request, res: express.Response) {
 
   const { resolution: maxResolution, audioStream } = await video.probeMaxQualityFile()
   const resolutions = await Hooks.wrapObject(
-    computeResolutionsToTranscode({ inputResolution: maxResolution, type: 'vod', includeInputResolution: true }),
+    computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false }),
     'filter:transcoding.manual.resolutions-to-transcode.result',
     body
   )

diff --git a/server/helpers/ffmpeg/ffprobe-utils.ts b/server/helpers/ffmpeg/ffprobe-utils.ts
index 7bcd27665829756d5b8b4706ff09c97573b20445..c45f9ec99c2850c5314842b65f9f9071d59290e8 100644 (file)
--- a/server/helpers/ffmpeg/ffprobe-utils.ts
+++ b/server/helpers/ffmpeg/ffprobe-utils.ts
@@ -91,11 +91,12 @@ async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
 // ---------------------------------------------------------------------------
 
 function computeResolutionsToTranscode (options: {
-  inputResolution: number
+  input: number
   type: 'vod' | 'live'
-  includeInputResolution: boolean
+  includeInput: boolean
+  strictLower: boolean
 }) {
-  const { inputResolution, type, includeInputResolution } = options
+  const { input, type, includeInput, strictLower } = options
 
   const configResolutions = type === 'vod'
     ? CONFIG.TRANSCODING.RESOLUTIONS
@@ -117,13 +118,18 @@ function computeResolutionsToTranscode (options: {
   ]
 
   for (const resolution of availableResolutions) {
-    if (configResolutions[resolution + 'p'] === true && inputResolution > resolution) {
-      resolutionsEnabled.add(resolution)
-    }
+    // Resolution not enabled
+    if (configResolutions[resolution + 'p'] !== true) continue
+    // Too big resolution for input file
+    if (input < resolution) continue
+    // We only want lower resolutions than input file
+    if (strictLower && input === resolution) continue
+
+    resolutionsEnabled.add(resolution)
   }
 
-  if (includeInputResolution) {
-    resolutionsEnabled.add(inputResolution)
+  if (includeInput) {
+    resolutionsEnabled.add(input)
   }
 
   return Array.from(resolutionsEnabled)

diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 728f096b5bd581acdb728390c9402b3964f01643..13c990a1e186b35c01a7c02054433f9dfe465093 100644 (file)
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -57,7 +57,7 @@ export class YoutubeDLCLI {
     }
   }
 
-  static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[]) {
+  static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) {
     /**
      * list of format selectors in order or preference
      * see https://github.com/ytdl-org/youtube-dl#format-selection
@@ -69,18 +69,26 @@ export class YoutubeDLCLI {
      *
      * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
      **/
-    const resolution = enabledResolutions.length === 0
-      ? VideoResolution.H_720P
-      : Math.max(...enabledResolutions)
-
-    return [
-      `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
-      `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
-      `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
-      `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
+
+    let result: string[] = []
+
+    if (!useBestFormat) {
+      const resolution = enabledResolutions.length === 0
+        ? VideoResolution.H_720P
+        : Math.max(...enabledResolutions)
+
+      result = [
+        `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
+        `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
+        `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
+      ]
+    }
+
+    return result.concat([
+      'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
       'best' // Ultimate fallback
-    ].join('/')
+    ]).join('/')
   }
 
   private constructor () {

diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index d585e9a95707ac80085131188e706bd752016a06..176cf3b6995ef977e068aa070ba9ea15be8f61cb 100644 (file)
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -21,7 +21,11 @@ const processOptions = {
 
 class YoutubeDLWrapper {
 
-  constructor (private readonly url: string = '', private readonly enabledResolutions: number[] = []) {
+  constructor (
+    private readonly url: string,
+    private readonly enabledResolutions: number[],
+    private readonly useBestFormat: boolean
+  ) {
 
   }
 
@@ -30,7 +34,7 @@ class YoutubeDLWrapper {
 
     const info = await youtubeDL.getInfo({
       url: this.url,
-      format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
       additionalYoutubeDLArgs: youtubeDLArgs,
       processOptions
     })
@@ -80,7 +84,7 @@ class YoutubeDLWrapper {
     try {
       await youtubeDL.download({
         url: this.url,
-        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
         output: pathWithoutExtension,
         timeout,
         processOptions

diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 40804e82e516db087834e8fa71800ce5804a33f3..4cde26aef968c4bb82500d9729542126a9b1dcc6 100644 (file)
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -2,6 +2,7 @@ import { Job } from 'bull'
 import { move, remove, stat } from 'fs-extra'
 import { retryTransactionWrapper } from '@server/helpers/database-utils'
 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
+import { CONFIG } from '@server/initializers/config'
 import { isPostImportVideoAccepted } from '@server/lib/moderation'
 import { generateWebTorrentVideoFilename } from '@server/lib/paths'
 import { Hooks } from '@server/lib/plugins/hooks'
@@ -25,7 +26,7 @@ import {
   VideoResolution,
   VideoState
 } from '@shared/models'
-import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, getVideoStreamDimensionsInfo } from '../../../helpers/ffmpeg'
+import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS } from '../../../helpers/ffmpeg'
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
@@ -80,7 +81,11 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul
 
   const options = { type: payload.type, videoImportId: videoImport.id }
 
-  const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
+  const youtubeDL = new YoutubeDLWrapper(
+    videoImport.targetUrl,
+    ServerConfigManager.Instance.getEnabledResolutions('vod'),
+    CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+  )
 
   return processFile(
     () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),

diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 890d34e3b8e64e10165e71676a2b3c745ca91e7c..4e5e979190758b3adc89f779fa9f2787c6ffc84f 100644 (file)
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -265,7 +265,7 @@ async function createLowerResolutionsJobs (options: {
 
   // Create transcoding jobs if there are enabled resolutions
   const resolutionsEnabled = await Hooks.wrapObject(
-    computeResolutionsToTranscode({ inputResolution: videoFileResolution, type: 'vod', includeInputResolution: false }),
+    computeResolutionsToTranscode({ input: videoFileResolution, type: 'vod', includeInput: false, strictLower: true }),
     'filter:transcoding.auto.resolutions-to-transcode.result',
     options
   )

diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts
index 3ac57fa440658751e077e4db7a8c4f9b2255ffc4..1410889a2dc1ba31f7419d4630c124911d573a70 100644 (file)
--- a/server/lib/live/live-manager.ts
+++ b/server/lib/live/live-manager.ts
@@ -456,10 +456,10 @@ class LiveManager {
   }
 
   private buildAllResolutionsToTranscode (originResolution: number) {
-    const includeInputResolution = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+    const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
 
     const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED
-      ? computeResolutionsToTranscode({ inputResolution: originResolution, type: 'live', includeInputResolution })
+      ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false })
       : []
 
     if (resolutionsEnabled.length === 0) {

diff --git a/server/lib/transcoding/transcoding.ts b/server/lib/transcoding/transcoding.ts
index 3681de994d281777b794d36316bb6ae1d1c9d8b5..070c7ebda77018f914158d5367d38a2c414a7fca 100644 (file)
--- a/server/lib/transcoding/transcoding.ts
+++ b/server/lib/transcoding/transcoding.ts
@@ -366,7 +366,7 @@ async function generateHlsPlaylistCommon (options: {
 function buildOriginalFileResolution (inputResolution: number) {
   if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) return toEven(inputResolution)
 
-  const resolutions = computeResolutionsToTranscode({ inputResolution, type: 'vod', includeInputResolution: false })
+  const resolutions = computeResolutionsToTranscode({ input: inputResolution, type: 'vod', includeInput: false, strictLower: false })
   if (resolutions.length === 0) return toEven(inputResolution)
 
   return Math.max(...resolutions)

diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
index cf9f7d0cbb029ce8d84c876274ca4b01bd565012..603e2d23444e1067b295743dfffa96895eb2b605 100644 (file)
--- a/server/tests/api/videos/video-imports.ts
+++ b/server/tests/api/videos/video-imports.ts
@@ -6,7 +6,7 @@ import { pathExists, readdir, remove } from 'fs-extra'
 import { join } from 'path'
 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
 import { areHttpImportTestsDisabled } from '@shared/core-utils'
-import { HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
+import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
 import {
   cleanupTests,
   createMultipleServers,
@@ -17,6 +17,7 @@ import {
   setDefaultVideoChannel,
   waitJobs
 } from '@shared/server-commands'
+import { DeepPartial } from '@shared/typescript-utils'
 
 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
   const videoHttp = await server.videos.get({ id: idHttp })
@@ -105,6 +106,16 @@ describe('Test video imports', function () {
         await setAccessTokensToServers(servers)
         await setDefaultVideoChannel(servers)
 
+        for (const server of servers) {
+          await server.config.updateExistingSubConfig({
+            newConfig: {
+              transcoding: {
+                alwaysTranscodeOriginalResolution: false
+              }
+            }
+          })
+        }
+
         await doubleFollow(servers[0], servers[1])
       })
 
@@ -306,10 +317,11 @@ describe('Test video imports', function () {
       it('Should import no HDR version on a HDR video', async function () {
         this.timeout(300_000)
 
-        const config = {
+        const config: DeepPartial<CustomConfig> = {
           transcoding: {
             enabled: true,
             resolutions: {
+              '0p': false,
               '144p': true,
               '240p': true,
               '360p': false,
@@ -321,19 +333,9 @@ describe('Test video imports', function () {
             },
             webtorrent: { enabled: true },
             hls: { enabled: false }
-          },
-          import: {
-            videos: {
-              http: {
-                enabled: true
-              },
-              torrent: {
-                enabled: true
-              }
-            }
           }
         }
-        await servers[0].config.updateCustomSubConfig({ newConfig: config })
+        await servers[0].config.updateExistingSubConfig({ newConfig: config })
 
         const attributes = {
           name: 'hdr video',
@@ -353,6 +355,76 @@ describe('Test video imports', function () {
         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
       })
 
+      it('Should not import resolution higher than enabled transcoding resolution', async function () {
+        this.timeout(300_000)
+
+        const config: DeepPartial<CustomConfig> = {
+          transcoding: {
+            enabled: true,
+            resolutions: {
+              '0p': false,
+              '144p': true,
+              '240p': false,
+              '360p': false,
+              '480p': false,
+              '720p': false,
+              '1080p': false,
+              '1440p': false,
+              '2160p': false
+            },
+            alwaysTranscodeOriginalResolution: false
+          }
+        }
+        await servers[0].config.updateExistingSubConfig({ newConfig: config })
+
+        const attributes = {
+          name: 'small resolution video',
+          targetUrl: FIXTURE_URLS.youtube,
+          channelId: servers[0].store.channel.id,
+          privacy: VideoPrivacy.PUBLIC
+        }
+        const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
+        const videoUUID = videoImported.uuid
+
+        await waitJobs(servers)
+
+        // test resolution
+        const video = await servers[0].videos.get({ id: videoUUID })
+        expect(video.name).to.equal('small resolution video')
+        expect(video.files).to.have.lengthOf(1)
+        expect(video.files[0].resolution.id).to.equal(144)
+      })
+
+      it('Should import resolution higher than enabled transcoding resolution', async function () {
+        this.timeout(300_000)
+
+        const config: DeepPartial<CustomConfig> = {
+          transcoding: {
+            alwaysTranscodeOriginalResolution: true
+          }
+        }
+        await servers[0].config.updateExistingSubConfig({ newConfig: config })
+
+        const attributes = {
+          name: 'bigger resolution video',
+          targetUrl: FIXTURE_URLS.youtube,
+          channelId: servers[0].store.channel.id,
+          privacy: VideoPrivacy.PUBLIC
+        }
+        const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
+        const videoUUID = videoImported.uuid
+
+        await waitJobs(servers)
+
+        // test resolution
+        const video = await servers[0].videos.get({ id: videoUUID })
+        expect(video.name).to.equal('bigger resolution video')
+
+        expect(video.files).to.have.lengthOf(2)
+        expect(video.files.find(f => f.resolution.id === 240)).to.exist
+        expect(video.files.find(f => f.resolution.id === 144)).to.exist
+      })
+
       it('Should import a peertube video', async function () {
         this.timeout(120_000)