server/helpers/youtube-dl/youtube-dl-cli.ts

   1 import execa from 'execa'
   2 import { ensureDir, pathExists, writeFile } from 'fs-extra'
   3 import { dirname, join } from 'path'
   4 import { CONFIG } from '@server/initializers/config'
   5 import { VideoResolution } from '@shared/models'
   6 import { logger, loggerTagsFactory } from '../logger'
   7 import { getProxy, isProxyEnabled } from '../proxy'
   8 import { isBinaryResponse, peertubeGot } from '../requests'
   9
  10 const lTags = loggerTagsFactory('youtube-dl')
  11
  12 const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
  13
  14 export class YoutubeDLCLI {
  15
  16   static async safeGet () {
  17     if (!await pathExists(youtubeDLBinaryPath)) {
  18       await ensureDir(dirname(youtubeDLBinaryPath))
  19
  20       await this.updateYoutubeDLBinary()
  21     }
  22
  23     return new YoutubeDLCLI()
  24   }
  25
  26   static async updateYoutubeDLBinary () {
  27     const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
  28
  29     logger.info('Updating youtubeDL binary from %s.', url, lTags())
  30
  31     const gotOptions = { context: { bodyKBLimit: 20_000 }, responseType: 'buffer' as 'buffer' }
  32
  33     try {
  34       let gotResult = await peertubeGot(url, gotOptions)
  35
  36       if (!isBinaryResponse(gotResult)) {
  37         const json = JSON.parse(gotResult.body.toString())
  38         const latest = json.filter(release => release.prerelease === false)[0]
  39         if (!latest) throw new Error('Cannot find latest release')
  40
  41         const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
  42         const releaseAsset = latest.assets.find(a => a.name === releaseName)
  43         if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
  44
  45         gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
  46       }
  47
  48       if (!isBinaryResponse(gotResult)) {
  49         throw new Error('Not a binary response')
  50       }
  51
  52       await writeFile(youtubeDLBinaryPath, gotResult.body)
  53
  54       logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
  55     } catch (err) {
  56       logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
  57     }
  58   }
  59
  60   static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) {
  61     /**
  62      * list of format selectors in order or preference
  63      * see https://github.com/ytdl-org/youtube-dl#format-selection
  64      *
  65      * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
  66      * of being able to do a "quick-transcode"
  67      * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
  68      * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
  69      *
  70      * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
  71      **/
  72
  73     let result: string[] = []
  74
  75     if (!useBestFormat) {
  76       const resolution = enabledResolutions.length === 0
  77         ? VideoResolution.H_720P
  78         : Math.max(...enabledResolutions)
  79
  80       result = [
  81         `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
  82         `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
  83         `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
  84       ]
  85     }
  86
  87     return result.concat([
  88       'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
  89       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
  90       'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
  91       'best' // Ultimate fallback
  92     ]).join('/')
  93   }
  94
  95   private constructor () {
  96
  97   }
  98
  99   download (options: {
 100     url: string
 101     format: string
 102     output: string
 103     processOptions: execa.NodeOptions
 104     timeout?: number
 105     additionalYoutubeDLArgs?: string[]
 106   }) {
 107     let args = options.additionalYoutubeDLArgs || []
 108     args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
 109
 110     return this.run({
 111       url: options.url,
 112       processOptions: options.processOptions,
 113       timeout: options.timeout,
 114       args
 115     })
 116   }
 117
 118   async getInfo (options: {
 119     url: string
 120     format: string
 121     processOptions: execa.NodeOptions
 122     additionalYoutubeDLArgs?: string[]
 123   }) {
 124     const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
 125
 126     const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
 127
 128     const data = await this.run({ url, args: completeArgs, processOptions })
 129     if (!data) return undefined
 130
 131     const info = data.map(d => JSON.parse(d))
 132
 133     return info.length === 1
 134       ? info[0]
 135       : info
 136   }
 137
 138   async getListInfo (options: {
 139     url: string
 140     latestVideosCount?: number
 141     processOptions: execa.NodeOptions
 142   }): Promise<{ upload_date: string, webpage_url: string }[]> {
 143     const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
 144
 145     if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
 146       // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
 147       additionalYoutubeDLArgs.push('--flat-playlist')
 148     }
 149
 150     if (options.latestVideosCount !== undefined) {
 151       additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
 152     }
 153
 154     const result = await this.getInfo({
 155       url: options.url,
 156       format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
 157       processOptions: options.processOptions,
 158       additionalYoutubeDLArgs
 159     })
 160
 161     if (!result) return result
 162     if (!Array.isArray(result)) return [ result ]
 163
 164     return result
 165   }
 166
 167   async getSubs (options: {
 168     url: string
 169     format: 'vtt'
 170     processOptions: execa.NodeOptions
 171   }) {
 172     const { url, format, processOptions } = options
 173
 174     const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
 175
 176     const data = await this.run({ url, args, processOptions })
 177     const files: string[] = []
 178
 179     const skipString = '[info] Writing video subtitles to: '
 180
 181     for (let i = 0, len = data.length; i < len; i++) {
 182       const line = data[i]
 183
 184       if (line.indexOf(skipString) === 0) {
 185         files.push(line.slice(skipString.length))
 186       }
 187     }
 188
 189     return files
 190   }
 191
 192   private async run (options: {
 193     url: string
 194     args: string[]
 195     timeout?: number
 196     processOptions: execa.NodeOptions
 197   }) {
 198     const { url, args, timeout, processOptions } = options
 199
 200     let completeArgs = this.wrapWithProxyOptions(args)
 201     completeArgs = this.wrapWithIPOptions(completeArgs)
 202     completeArgs = this.wrapWithFFmpegOptions(completeArgs)
 203
 204     const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
 205     const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
 206
 207     if (timeout) {
 208       setTimeout(() => subProcess.cancel(), timeout)
 209     }
 210
 211     const output = await subProcess
 212
 213     logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
 214
 215     return output.stdout
 216       ? output.stdout.trim().split(/\r?\n/)
 217       : undefined
 218   }
 219
 220   private wrapWithProxyOptions (args: string[]) {
 221     if (isProxyEnabled()) {
 222       logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
 223
 224       return [ '--proxy', getProxy() ].concat(args)
 225     }
 226
 227     return args
 228   }
 229
 230   private wrapWithIPOptions (args: string[]) {
 231     if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
 232       logger.debug('Force ipv4 for YoutubeDL')
 233
 234       return [ '--force-ipv4' ].concat(args)
 235     }
 236
 237     return args
 238   }
 239
 240   private wrapWithFFmpegOptions (args: string[]) {
 241     if (process.env.FFMPEG_PATH) {
 242       logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
 243
 244       return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
 245     }
 246
 247     return args
 248   }
 249 }