server/helpers/youtube-dl/youtube-dl-cli.ts

   1 import execa from 'execa'
   2 import { pathExists, writeFile } from 'fs-extra'
   3 import { join } from 'path'
   4 import { CONFIG } from '@server/initializers/config'
   5 import { VideoResolution } from '@shared/models'
   6 import { logger, loggerTagsFactory } from '../logger'
   7 import { getProxy, isProxyEnabled } from '../proxy'
   8 import { isBinaryResponse, peertubeGot } from '../requests'
   9
  10 const lTags = loggerTagsFactory('youtube-dl')
  11
  12 const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
  13
  14 export class YoutubeDLCLI {
  15
  16   static async safeGet () {
  17     if (!await pathExists(youtubeDLBinaryPath)) {
  18       await this.updateYoutubeDLBinary()
  19     }
  20
  21     return new YoutubeDLCLI()
  22   }
  23
  24   static async updateYoutubeDLBinary () {
  25     const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
  26
  27     logger.info('Updating youtubeDL binary from %s.', url, lTags())
  28
  29     const gotOptions = { context: { bodyKBLimit: 20_000 }, responseType: 'buffer' as 'buffer' }
  30
  31     try {
  32       let gotResult = await peertubeGot(url, gotOptions)
  33
  34       if (!isBinaryResponse(gotResult)) {
  35         const json = JSON.parse(gotResult.body.toString())
  36         const latest = json.filter(release => release.prerelease === false)[0]
  37         if (!latest) throw new Error('Cannot find latest release')
  38
  39         const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
  40         const releaseAsset = latest.assets.find(a => a.name === releaseName)
  41         if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
  42
  43         gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
  44       }
  45
  46       if (!isBinaryResponse(gotResult)) {
  47         throw new Error('Not a binary response')
  48       }
  49
  50       await writeFile(youtubeDLBinaryPath, gotResult.body)
  51
  52       logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
  53     } catch (err) {
  54       logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
  55     }
  56   }
  57
  58   static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[]) {
  59     /**
  60      * list of format selectors in order or preference
  61      * see https://github.com/ytdl-org/youtube-dl#format-selection
  62      *
  63      * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
  64      * of being able to do a "quick-transcode"
  65      * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
  66      * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
  67      *
  68      * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
  69      **/
  70     const resolution = enabledResolutions.length === 0
  71       ? VideoResolution.H_720P
  72       : Math.max(...enabledResolutions)
  73
  74     return [
  75       `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
  76       `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
  77       `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
  78       `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
  79       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
  80       'best' // Ultimate fallback
  81     ].join('/')
  82   }
  83
  84   private constructor () {
  85
  86   }
  87
  88   download (options: {
  89     url: string
  90     format: string
  91     output: string
  92     processOptions: execa.NodeOptions
  93     additionalYoutubeDLArgs?: string[]
  94   }) {
  95     return this.run({
  96       url: options.url,
  97       processOptions: options.processOptions,
  98       args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
  99     })
 100   }
 101
 102   async getInfo (options: {
 103     url: string
 104     format: string
 105     processOptions: execa.NodeOptions
 106     additionalYoutubeDLArgs?: string[]
 107   }) {
 108     const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
 109
 110     const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
 111
 112     const data = await this.run({ url, args: completeArgs, processOptions })
 113     const info = data.map(this.parseInfo)
 114
 115     return info.length === 1
 116       ? info[0]
 117       : info
 118   }
 119
 120   async getSubs (options: {
 121     url: string
 122     format: 'vtt'
 123     processOptions: execa.NodeOptions
 124   }) {
 125     const { url, format, processOptions } = options
 126
 127     const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
 128
 129     const data = await this.run({ url, args, processOptions })
 130     const files: string[] = []
 131
 132     const skipString = '[info] Writing video subtitles to: '
 133
 134     for (let i = 0, len = data.length; i < len; i++) {
 135       const line = data[i]
 136
 137       if (line.indexOf(skipString) === 0) {
 138         files.push(line.slice(skipString.length))
 139       }
 140     }
 141
 142     return files
 143   }
 144
 145   private async run (options: {
 146     url: string
 147     args: string[]
 148     processOptions: execa.NodeOptions
 149   }) {
 150     const { url, args, processOptions } = options
 151
 152     let completeArgs = this.wrapWithProxyOptions(args)
 153     completeArgs = this.wrapWithIPOptions(completeArgs)
 154     completeArgs = this.wrapWithFFmpegOptions(completeArgs)
 155
 156     const output = await execa('python', [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
 157
 158     logger.debug('Runned youtube-dl command.', { command: output.command, stdout: output.stdout, ...lTags() })
 159
 160     return output.stdout
 161       ? output.stdout.trim().split(/\r?\n/)
 162       : undefined
 163   }
 164
 165   private wrapWithProxyOptions (args: string[]) {
 166     if (isProxyEnabled()) {
 167       logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
 168
 169       return [ '--proxy', getProxy() ].concat(args)
 170     }
 171
 172     return args
 173   }
 174
 175   private wrapWithIPOptions (args: string[]) {
 176     if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
 177       logger.debug('Force ipv4 for YoutubeDL')
 178
 179       return [ '--force-ipv4' ].concat(args)
 180     }
 181
 182     return args
 183   }
 184
 185   private wrapWithFFmpegOptions (args: string[]) {
 186     if (process.env.FFMPEG_PATH) {
 187       logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
 188
 189       return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
 190     }
 191
 192     return args
 193   }
 194
 195   private parseInfo (data: string) {
 196     return JSON.parse(data)
 197   }
 198 }