server/helpers/youtube-dl/youtube-dl-cli.ts

   1 import execa from 'execa'
   2 import { pathExists, writeFile } from 'fs-extra'
   3 import { join } from 'path'
   4 import { CONFIG } from '@server/initializers/config'
   5 import { VideoResolution } from '@shared/models'
   6 import { logger, loggerTagsFactory } from '../logger'
   7 import { getProxy, isProxyEnabled } from '../proxy'
   8 import { isBinaryResponse, peertubeGot } from '../requests'
   9
  10 const lTags = loggerTagsFactory('youtube-dl')
  11
  12 const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
  13
  14 export class YoutubeDLCLI {
  15
  16   static async safeGet () {
  17     if (!await pathExists(youtubeDLBinaryPath)) {
  18       await this.updateYoutubeDLBinary()
  19     }
  20
  21     return new YoutubeDLCLI()
  22   }
  23
  24   static async updateYoutubeDLBinary () {
  25     const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
  26
  27     logger.info('Updating youtubeDL binary from %s.', url, lTags())
  28
  29     const gotOptions = { context: { bodyKBLimit: 20_000 }, responseType: 'buffer' as 'buffer' }
  30
  31     try {
  32       let gotResult = await peertubeGot(url, gotOptions)
  33
  34       if (!isBinaryResponse(gotResult)) {
  35         const json = JSON.parse(gotResult.body.toString())
  36         const latest = json.filter(release => release.prerelease === false)[0]
  37         if (!latest) throw new Error('Cannot find latest release')
  38
  39         const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
  40         const releaseAsset = latest.assets.find(a => a.name === releaseName)
  41         if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
  42
  43         gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
  44       }
  45
  46       if (!isBinaryResponse(gotResult)) {
  47         throw new Error('Not a binary response')
  48       }
  49
  50       await writeFile(youtubeDLBinaryPath, gotResult.body)
  51
  52       logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
  53     } catch (err) {
  54       logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
  55     }
  56   }
  57
  58   static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[]) {
  59     /**
  60      * list of format selectors in order or preference
  61      * see https://github.com/ytdl-org/youtube-dl#format-selection
  62      *
  63      * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
  64      * of being able to do a "quick-transcode"
  65      * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
  66      * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
  67      *
  68      * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
  69      **/
  70     const resolution = enabledResolutions.length === 0
  71       ? VideoResolution.H_720P
  72       : Math.max(...enabledResolutions)
  73
  74     return [
  75       `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
  76       `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
  77       `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
  78       `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
  79       'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
  80       'best' // Ultimate fallback
  81     ].join('/')
  82   }
  83
  84   private constructor () {
  85
  86   }
  87
  88   download (options: {
  89     url: string
  90     format: string
  91     output: string
  92     processOptions: execa.NodeOptions
  93     timeout?: number
  94     additionalYoutubeDLArgs?: string[]
  95   }) {
  96     return this.run({
  97       url: options.url,
  98       processOptions: options.processOptions,
  99       timeout: options.timeout,
 100       args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
 101     })
 102   }
 103
 104   async getInfo (options: {
 105     url: string
 106     format: string
 107     processOptions: execa.NodeOptions
 108     additionalYoutubeDLArgs?: string[]
 109   }) {
 110     const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
 111
 112     const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
 113
 114     const data = await this.run({ url, args: completeArgs, processOptions })
 115     const info = data.map(this.parseInfo)
 116
 117     return info.length === 1
 118       ? info[0]
 119       : info
 120   }
 121
 122   async getSubs (options: {
 123     url: string
 124     format: 'vtt'
 125     processOptions: execa.NodeOptions
 126   }) {
 127     const { url, format, processOptions } = options
 128
 129     const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
 130
 131     const data = await this.run({ url, args, processOptions })
 132     const files: string[] = []
 133
 134     const skipString = '[info] Writing video subtitles to: '
 135
 136     for (let i = 0, len = data.length; i < len; i++) {
 137       const line = data[i]
 138
 139       if (line.indexOf(skipString) === 0) {
 140         files.push(line.slice(skipString.length))
 141       }
 142     }
 143
 144     return files
 145   }
 146
 147   private async run (options: {
 148     url: string
 149     args: string[]
 150     timeout?: number
 151     processOptions: execa.NodeOptions
 152   }) {
 153     const { url, args, timeout, processOptions } = options
 154
 155     let completeArgs = this.wrapWithProxyOptions(args)
 156     completeArgs = this.wrapWithIPOptions(completeArgs)
 157     completeArgs = this.wrapWithFFmpegOptions(completeArgs)
 158
 159     const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
 160     const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
 161
 162     if (timeout) {
 163       setTimeout(() => subProcess.cancel(), timeout)
 164     }
 165
 166     const output = await subProcess
 167
 168     logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
 169
 170     return output.stdout
 171       ? output.stdout.trim().split(/\r?\n/)
 172       : undefined
 173   }
 174
 175   private wrapWithProxyOptions (args: string[]) {
 176     if (isProxyEnabled()) {
 177       logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
 178
 179       return [ '--proxy', getProxy() ].concat(args)
 180     }
 181
 182     return args
 183   }
 184
 185   private wrapWithIPOptions (args: string[]) {
 186     if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
 187       logger.debug('Force ipv4 for YoutubeDL')
 188
 189       return [ '--force-ipv4' ].concat(args)
 190     }
 191
 192     return args
 193   }
 194
 195   private wrapWithFFmpegOptions (args: string[]) {
 196     if (process.env.FFMPEG_PATH) {
 197       logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
 198
 199       return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
 200     }
 201
 202     return args
 203   }
 204
 205   private parseInfo (data: string) {
 206     return JSON.parse(data)
 207   }
 208 }