server/helpers/youtube-dl.ts

   1 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
   2 import { logger } from './logger'
   3 import { generateVideoImportTmpPath } from './utils'
   4 import { getEnabledResolutions } from '../lib/video-transcoding'
   5 import { join } from 'path'
   6 import { peertubeTruncate, root } from './core-utils'
   7 import { ensureDir, remove, writeFile } from 'fs-extra'
   8 import * as request from 'request'
   9 import { createWriteStream } from 'fs'
  10 import { CONFIG } from '@server/initializers/config'
  11 import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
  12 import { VideoResolution } from '../../shared/models/videos'
  13
  14 export type YoutubeDLInfo = {
  15   name?: string
  16   description?: string
  17   category?: number
  18   language?: string
  19   licence?: number
  20   nsfw?: boolean
  21   tags?: string[]
  22   thumbnailUrl?: string
  23   ext?: string
  24   mergeExt?: string
  25   originallyPublishedAt?: Date
  26 }
  27
  28 export type YoutubeDLSubs = {
  29   language: string
  30   filename: string
  31   path: string
  32 }[]
  33
  34 const processOptions = {
  35   maxBuffer: 1024 * 1024 * 10 // 10MB
  36 }
  37
  38 function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
  39   return new Promise<YoutubeDLInfo>((res, rej) => {
  40     let args = opts || [ '-j', '--flat-playlist' ]
  41
  42     if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
  43       args.push('--force-ipv4')
  44     }
  45
  46     args = wrapWithProxyOptions(args)
  47     args = [ '-f', getYoutubeDLVideoFormat() ].concat(args)
  48
  49     safeGetYoutubeDL()
  50       .then(youtubeDL => {
  51         youtubeDL.getInfo(url, args, processOptions, (err, info) => {
  52           if (err) return rej(err)
  53           if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
  54           if (info.format_id?.includes('+')) {
  55             // this is a merge format and its extension will be appended
  56             if (info.ext === 'mp4') {
  57               info.mergeExt = 'mp4'
  58             } else {
  59               info.mergeExt = 'mkv'
  60             }
  61           }
  62
  63           const obj = buildVideoInfo(normalizeObject(info))
  64           if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
  65
  66           return res(obj)
  67         })
  68       })
  69       .catch(err => rej(err))
  70   })
  71 }
  72
  73 function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
  74   return new Promise<YoutubeDLSubs>((res, rej) => {
  75     const cwd = CONFIG.STORAGE.TMP_DIR
  76     const options = opts || { all: true, format: 'vtt', cwd }
  77
  78     safeGetYoutubeDL()
  79       .then(youtubeDL => {
  80         youtubeDL.getSubs(url, options, (err, files) => {
  81           if (err) return rej(err)
  82           if (!files) return []
  83
  84           logger.debug('Get subtitles from youtube dl.', { url, files })
  85
  86           const subtitles = files.reduce((acc, filename) => {
  87             const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
  88             if (!matched || !matched[1]) return acc
  89
  90             return [
  91               ...acc,
  92               {
  93                 language: matched[1],
  94                 path: join(cwd, filename),
  95                 filename
  96               }
  97             ]
  98           }, [])
  99
 100           return res(subtitles)
 101         })
 102       })
 103       .catch(err => rej(err))
 104   })
 105 }
 106
 107 function getYoutubeDLVideoFormat () {
 108   /**
 109    * list of format selectors in order or preference
 110    * see https://github.com/ytdl-org/youtube-dl#format-selection
 111    *
 112    * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
 113    * of being able to do a "quick-transcode"
 114    * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
 115    * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
 116    *
 117    * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
 118    **/
 119   const enabledResolutions = getEnabledResolutions('vod')
 120   const resolution = enabledResolutions.length === 0
 121     ? VideoResolution.H_720P
 122     : Math.max(...enabledResolutions)
 123
 124   return [
 125     `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
 126     `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
 127     `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
 128     `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
 129     'best[vcodec!*=av01][vcodec!*=vp9.2]' // case fallback
 130   ].join('/')
 131 }
 132
 133 function downloadYoutubeDLVideo (url: string, extension: string, timeout: number, mergeExtension?: string) {
 134   const path = generateVideoImportTmpPath(url, extension)
 135   const finalPath = mergeExtension ? path.replace(new RegExp(`${extension}$`), mergeExtension) : path
 136   let timer
 137
 138   logger.info('Importing youtubeDL video %s to %s', url, finalPath)
 139
 140   let options = [ '-f', getYoutubeDLVideoFormat(), '-o', path ]
 141   options = wrapWithProxyOptions(options)
 142
 143   if (process.env.FFMPEG_PATH) {
 144     options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
 145   }
 146
 147   return new Promise<string>((res, rej) => {
 148     safeGetYoutubeDL()
 149       .then(youtubeDL => {
 150         youtubeDL.exec(url, options, processOptions, err => {
 151           clearTimeout(timer)
 152
 153           if (err) {
 154             remove(path)
 155               .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
 156
 157             return rej(err)
 158           }
 159
 160           return res(finalPath)
 161         })
 162
 163         timer = setTimeout(() => {
 164           const err = new Error('YoutubeDL download timeout.')
 165
 166           remove(path)
 167             .finally(() => rej(err))
 168             .catch(err => {
 169               logger.error('Cannot remove %s in youtubeDL timeout.', path, { err })
 170               return rej(err)
 171             })
 172         }, timeout)
 173       })
 174       .catch(err => rej(err))
 175   })
 176 }
 177
 178 // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
 179 // We rewrote it to avoid sync calls
 180 async function updateYoutubeDLBinary () {
 181   logger.info('Updating youtubeDL binary.')
 182
 183   const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
 184   const bin = join(binDirectory, 'youtube-dl')
 185   const detailsPath = join(binDirectory, 'details')
 186   const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl'
 187
 188   await ensureDir(binDirectory)
 189
 190   return new Promise(res => {
 191     request.get(url, { followRedirect: false }, (err, result) => {
 192       if (err) {
 193         logger.error('Cannot update youtube-dl.', { err })
 194         return res()
 195       }
 196
 197       if (result.statusCode !== HttpStatusCode.FOUND_302) {
 198         logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
 199         return res()
 200       }
 201
 202       const url = result.headers.location
 203       const downloadFile = request.get(url)
 204       const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
 205
 206       downloadFile.on('response', result => {
 207         if (result.statusCode !== HttpStatusCode.OK_200) {
 208           logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
 209           return res()
 210         }
 211
 212         const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => {
 213           logger.error('youtube-dl update error in write stream', { err })
 214           return res()
 215         })
 216
 217         downloadFile.pipe(writeStream)
 218       })
 219
 220       downloadFile.on('error', err => {
 221         logger.error('youtube-dl update error.', { err })
 222         return res()
 223       })
 224
 225       downloadFile.on('end', () => {
 226         const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
 227         writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
 228           if (err) {
 229             logger.error('youtube-dl update error: cannot write details.', { err })
 230             return res()
 231           }
 232
 233           logger.info('youtube-dl updated to version %s.', newVersion)
 234           return res()
 235         })
 236       })
 237     })
 238   })
 239 }
 240
 241 async function safeGetYoutubeDL () {
 242   let youtubeDL
 243
 244   try {
 245     youtubeDL = require('youtube-dl')
 246   } catch (e) {
 247     // Download binary
 248     await updateYoutubeDLBinary()
 249     youtubeDL = require('youtube-dl')
 250   }
 251
 252   return youtubeDL
 253 }
 254
 255 function buildOriginallyPublishedAt (obj: any) {
 256   let originallyPublishedAt: Date = null
 257
 258   const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
 259   if (uploadDateMatcher) {
 260     originallyPublishedAt = new Date()
 261     originallyPublishedAt.setHours(0, 0, 0, 0)
 262
 263     const year = parseInt(uploadDateMatcher[1], 10)
 264     // Month starts from 0
 265     const month = parseInt(uploadDateMatcher[2], 10) - 1
 266     const day = parseInt(uploadDateMatcher[3], 10)
 267
 268     originallyPublishedAt.setFullYear(year, month, day)
 269   }
 270
 271   return originallyPublishedAt
 272 }
 273
 274 // ---------------------------------------------------------------------------
 275
 276 export {
 277   updateYoutubeDLBinary,
 278   getYoutubeDLVideoFormat,
 279   downloadYoutubeDLVideo,
 280   getYoutubeDLSubs,
 281   getYoutubeDLInfo,
 282   safeGetYoutubeDL,
 283   buildOriginallyPublishedAt
 284 }
 285
 286 // ---------------------------------------------------------------------------
 287
 288 function normalizeObject (obj: any) {
 289   const newObj: any = {}
 290
 291   for (const key of Object.keys(obj)) {
 292     // Deprecated key
 293     if (key === 'resolution') continue
 294
 295     const value = obj[key]
 296
 297     if (typeof value === 'string') {
 298       newObj[key] = value.normalize()
 299     } else {
 300       newObj[key] = value
 301     }
 302   }
 303
 304   return newObj
 305 }
 306
 307 function buildVideoInfo (obj: any): YoutubeDLInfo {
 308   return {
 309     name: titleTruncation(obj.title),
 310     description: descriptionTruncation(obj.description),
 311     category: getCategory(obj.categories),
 312     licence: getLicence(obj.license),
 313     language: getLanguage(obj.language),
 314     nsfw: isNSFW(obj),
 315     tags: getTags(obj.tags),
 316     thumbnailUrl: obj.thumbnail || undefined,
 317     originallyPublishedAt: buildOriginallyPublishedAt(obj),
 318     ext: obj.ext,
 319     mergeExt: obj.mergeExt
 320   }
 321 }
 322
 323 function titleTruncation (title: string) {
 324   return peertubeTruncate(title, {
 325     length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
 326     separator: /,? +/,
 327     omission: ' […]'
 328   })
 329 }
 330
 331 function descriptionTruncation (description: string) {
 332   if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
 333
 334   return peertubeTruncate(description, {
 335     length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
 336     separator: /,? +/,
 337     omission: ' […]'
 338   })
 339 }
 340
 341 function isNSFW (info: any) {
 342   return info.age_limit && info.age_limit >= 16
 343 }
 344
 345 function getTags (tags: any) {
 346   if (Array.isArray(tags) === false) return []
 347
 348   return tags
 349     .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
 350     .map(t => t.normalize())
 351     .slice(0, 5)
 352 }
 353
 354 function getLicence (licence: string) {
 355   if (!licence) return undefined
 356
 357   if (licence.includes('Creative Commons Attribution')) return 1
 358
 359   for (const key of Object.keys(VIDEO_LICENCES)) {
 360     const peertubeLicence = VIDEO_LICENCES[key]
 361     if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
 362   }
 363
 364   return undefined
 365 }
 366
 367 function getCategory (categories: string[]) {
 368   if (!categories) return undefined
 369
 370   const categoryString = categories[0]
 371   if (!categoryString || typeof categoryString !== 'string') return undefined
 372
 373   if (categoryString === 'News & Politics') return 11
 374
 375   for (const key of Object.keys(VIDEO_CATEGORIES)) {
 376     const category = VIDEO_CATEGORIES[key]
 377     if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
 378   }
 379
 380   return undefined
 381 }
 382
 383 function getLanguage (language: string) {
 384   return VIDEO_LANGUAGES[language] ? language : undefined
 385 }
 386
 387 function wrapWithProxyOptions (options: string[]) {
 388   if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
 389     logger.debug('Using proxy for YoutubeDL')
 390
 391     return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options)
 392   }
 393
 394   return options
 395 }