Merge remote-tracking branch 'weblate/develop' into develop

[github/Chocobozzz/PeerTube.git] / server / helpers / youtube-dl / youtube-dl-cli.ts

diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 13c990a1e186b35c01a7c02054433f9dfe465093..765038cea29cd9e6b9ebee0e1663c81f1422aa12 100644 (file)
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -6,6 +6,7 @@ import { VideoResolution } from '@shared/models'
 import { logger, loggerTagsFactory } from '../logger'
 import { getProxy, isProxyEnabled } from '../proxy'
 import { isBinaryResponse, peertubeGot } from '../requests'
+import { OptionsOfBufferResponseBody } from 'got/dist/source'
 
 const lTags = loggerTagsFactory('youtube-dl')
 
@@ -28,7 +29,16 @@ export class YoutubeDLCLI {
 
     logger.info('Updating youtubeDL binary from %s.', url, lTags())
 
-    const gotOptions = { context: { bodyKBLimit: 20_000 }, responseType: 'buffer' as 'buffer' }
+    const gotOptions: OptionsOfBufferResponseBody = {
+      context: { bodyKBLimit: 20_000 },
+      responseType: 'buffer' as 'buffer'
+    }
+
+    if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
+      gotOptions.headers = {
+        authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
+      }
+    }
 
     try {
       let gotResult = await peertubeGot(url, gotOptions)
@@ -87,6 +97,7 @@ export class YoutubeDLCLI {
     return result.concat([
       'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+      'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
       'best' // Ultimate fallback
     ]).join('/')
   }
@@ -103,11 +114,14 @@ export class YoutubeDLCLI {
     timeout?: number
     additionalYoutubeDLArgs?: string[]
   }) {
+    let args = options.additionalYoutubeDLArgs || []
+    args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
+
     return this.run({
       url: options.url,
       processOptions: options.processOptions,
       timeout: options.timeout,
-      args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
+      args
     })
   }
 
@@ -122,13 +136,44 @@ export class YoutubeDLCLI {
     const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
 
     const data = await this.run({ url, args: completeArgs, processOptions })
-    const info = data.map(this.parseInfo)
+    if (!data) return undefined
+
+    const info = data.map(d => JSON.parse(d))
 
     return info.length === 1
       ? info[0]
       : info
   }
 
+  async getListInfo (options: {
+    url: string
+    latestVideosCount?: number
+    processOptions: execa.NodeOptions
+  }): Promise<{ upload_date: string, webpage_url: string }[]> {
+    const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+
+    if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
+      // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
+      additionalYoutubeDLArgs.push('--flat-playlist')
+    }
+
+    if (options.latestVideosCount !== undefined) {
+      additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
+    }
+
+    const result = await this.getInfo({
+      url: options.url,
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
+      processOptions: options.processOptions,
+      additionalYoutubeDLArgs
+    })
+
+    if (!result) return result
+    if (!Array.isArray(result)) return [ result ]
+
+    return result
+  }
+
   async getSubs (options: {
     url: string
     format: 'vtt'
@@ -175,7 +220,7 @@ export class YoutubeDLCLI {
 
     const output = await subProcess
 
-    logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
+    logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
 
     return output.stdout
       ? output.stdout.trim().split(/\r?\n/)
@@ -211,8 +256,4 @@ export class YoutubeDLCLI {
 
     return args
   }
-
-  private parseInfo (data: string) {
-    return JSON.parse(data)
-  }
 }