server/helpers/youtube-dl.ts

   1 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
   2 import { logger } from './logger'
   3 import { generateVideoImportTmpPath } from './utils'
   4 import { join } from 'path'
   5 import { peertubeTruncate, root } from './core-utils'
   6 import { ensureDir, remove, writeFile } from 'fs-extra'
   7 import * as request from 'request'
   8 import { createWriteStream } from 'fs'
   9 import { CONFIG } from '@server/initializers/config'
  10
  11 export type YoutubeDLInfo = {
  12   name?: string
  13   description?: string
  14   category?: number
  15   language?: string
  16   licence?: number
  17   nsfw?: boolean
  18   tags?: string[]
  19   thumbnailUrl?: string
  20   fileExt?: string
  21   originallyPublishedAt?: Date
  22 }
  23
  24 export type YoutubeDLSubs = {
  25   language: string
  26   filename: string
  27   path: string
  28 }[]
  29
  30 const processOptions = {
  31   maxBuffer: 1024 * 1024 * 10 // 10MB
  32 }
  33
  34 function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
  35   return new Promise<YoutubeDLInfo>((res, rej) => {
  36     let args = opts || [ '-j', '--flat-playlist' ]
  37     args = wrapWithProxyOptions(args)
  38
  39     safeGetYoutubeDL()
  40       .then(youtubeDL => {
  41         youtubeDL.getInfo(url, args, processOptions, (err, info) => {
  42           if (err) return rej(err)
  43           if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
  44
  45           const obj = buildVideoInfo(normalizeObject(info))
  46           if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
  47
  48           return res(obj)
  49         })
  50       })
  51       .catch(err => rej(err))
  52   })
  53 }
  54
  55 function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
  56   return new Promise<YoutubeDLSubs>((res, rej) => {
  57     const cwd = CONFIG.STORAGE.TMP_DIR
  58     const options = opts || { all: true, format: 'vtt', cwd }
  59
  60     safeGetYoutubeDL()
  61       .then(youtubeDL => {
  62         youtubeDL.getSubs(url, options, (err, files) => {
  63           if (err) return rej(err)
  64
  65           logger.debug('Get subtitles from youtube dl.', { url, files })
  66
  67           const subtitles = files.reduce((acc, filename) => {
  68             const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
  69
  70             if (matched[1]) {
  71               return [
  72                 ...acc,
  73                 {
  74                   language: matched[1],
  75                   path: join(cwd, filename),
  76                   filename
  77                 }
  78               ]
  79             }
  80           }, [])
  81
  82           return res(subtitles)
  83         })
  84       })
  85       .catch(err => rej(err))
  86   })
  87 }
  88
  89 function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
  90   const path = generateVideoImportTmpPath(url, extension)
  91   let timer
  92
  93   logger.info('Importing youtubeDL video %s to %s', url, path)
  94
  95   let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
  96   options = wrapWithProxyOptions(options)
  97
  98   if (process.env.FFMPEG_PATH) {
  99     options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
 100   }
 101
 102   return new Promise<string>((res, rej) => {
 103     safeGetYoutubeDL()
 104       .then(youtubeDL => {
 105         youtubeDL.exec(url, options, processOptions, err => {
 106           clearTimeout(timer)
 107
 108           if (err) {
 109             remove(path)
 110               .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
 111
 112             return rej(err)
 113           }
 114
 115           return res(path)
 116         })
 117
 118         timer = setTimeout(() => {
 119           const err = new Error('YoutubeDL download timeout.')
 120
 121           remove(path)
 122             .finally(() => rej(err))
 123             .catch(err => {
 124               logger.error('Cannot remove %s in youtubeDL timeout.', path, { err })
 125               return rej(err)
 126             })
 127         }, timeout)
 128       })
 129       .catch(err => rej(err))
 130   })
 131 }
 132
 133 // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
 134 // We rewrote it to avoid sync calls
 135 async function updateYoutubeDLBinary () {
 136   logger.info('Updating youtubeDL binary.')
 137
 138   const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
 139   const bin = join(binDirectory, 'youtube-dl')
 140   const detailsPath = join(binDirectory, 'details')
 141   const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
 142
 143   await ensureDir(binDirectory)
 144
 145   return new Promise(res => {
 146     request.get(url, { followRedirect: false }, (err, result) => {
 147       if (err) {
 148         logger.error('Cannot update youtube-dl.', { err })
 149         return res()
 150       }
 151
 152       if (result.statusCode !== 302) {
 153         logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
 154         return res()
 155       }
 156
 157       const url = result.headers.location
 158       const downloadFile = request.get(url)
 159       const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
 160
 161       downloadFile.on('response', result => {
 162         if (result.statusCode !== 200) {
 163           logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
 164           return res()
 165         }
 166
 167         downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
 168       })
 169
 170       downloadFile.on('error', err => {
 171         logger.error('youtube-dl update error.', { err })
 172         return res()
 173       })
 174
 175       downloadFile.on('end', () => {
 176         const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
 177         writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
 178           if (err) {
 179             logger.error('youtube-dl update error: cannot write details.', { err })
 180             return res()
 181           }
 182
 183           logger.info('youtube-dl updated to version %s.', newVersion)
 184           return res()
 185         })
 186       })
 187     })
 188   })
 189 }
 190
 191 async function safeGetYoutubeDL () {
 192   let youtubeDL
 193
 194   try {
 195     youtubeDL = require('youtube-dl')
 196   } catch (e) {
 197     // Download binary
 198     await updateYoutubeDLBinary()
 199     youtubeDL = require('youtube-dl')
 200   }
 201
 202   return youtubeDL
 203 }
 204
 205 function buildOriginallyPublishedAt (obj: any) {
 206   let originallyPublishedAt: Date = null
 207
 208   const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
 209   if (uploadDateMatcher) {
 210     originallyPublishedAt = new Date()
 211     originallyPublishedAt.setHours(0, 0, 0, 0)
 212
 213     const year = parseInt(uploadDateMatcher[1], 10)
 214     // Month starts from 0
 215     const month = parseInt(uploadDateMatcher[2], 10) - 1
 216     const day = parseInt(uploadDateMatcher[3], 10)
 217
 218     originallyPublishedAt.setFullYear(year, month, day)
 219   }
 220
 221   return originallyPublishedAt
 222 }
 223
 224 // ---------------------------------------------------------------------------
 225
 226 export {
 227   updateYoutubeDLBinary,
 228   downloadYoutubeDLVideo,
 229   getYoutubeDLSubs,
 230   getYoutubeDLInfo,
 231   safeGetYoutubeDL,
 232   buildOriginallyPublishedAt
 233 }
 234
 235 // ---------------------------------------------------------------------------
 236
 237 function normalizeObject (obj: any) {
 238   const newObj: any = {}
 239
 240   for (const key of Object.keys(obj)) {
 241     // Deprecated key
 242     if (key === 'resolution') continue
 243
 244     const value = obj[key]
 245
 246     if (typeof value === 'string') {
 247       newObj[key] = value.normalize()
 248     } else {
 249       newObj[key] = value
 250     }
 251   }
 252
 253   return newObj
 254 }
 255
 256 function buildVideoInfo (obj: any): YoutubeDLInfo {
 257   return {
 258     name: titleTruncation(obj.title),
 259     description: descriptionTruncation(obj.description),
 260     category: getCategory(obj.categories),
 261     licence: getLicence(obj.license),
 262     language: getLanguage(obj.language),
 263     nsfw: isNSFW(obj),
 264     tags: getTags(obj.tags),
 265     thumbnailUrl: obj.thumbnail || undefined,
 266     originallyPublishedAt: buildOriginallyPublishedAt(obj),
 267     fileExt: obj.ext
 268   }
 269 }
 270
 271 function titleTruncation (title: string) {
 272   return peertubeTruncate(title, {
 273     length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
 274     separator: /,? +/,
 275     omission: ' […]'
 276   })
 277 }
 278
 279 function descriptionTruncation (description: string) {
 280   if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
 281
 282   return peertubeTruncate(description, {
 283     length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
 284     separator: /,? +/,
 285     omission: ' […]'
 286   })
 287 }
 288
 289 function isNSFW (info: any) {
 290   return info.age_limit && info.age_limit >= 16
 291 }
 292
 293 function getTags (tags: any) {
 294   if (Array.isArray(tags) === false) return []
 295
 296   return tags
 297     .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
 298     .map(t => t.normalize())
 299     .slice(0, 5)
 300 }
 301
 302 function getLicence (licence: string) {
 303   if (!licence) return undefined
 304
 305   if (licence.includes('Creative Commons Attribution')) return 1
 306
 307   for (const key of Object.keys(VIDEO_LICENCES)) {
 308     const peertubeLicence = VIDEO_LICENCES[key]
 309     if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
 310   }
 311
 312   return undefined
 313 }
 314
 315 function getCategory (categories: string[]) {
 316   if (!categories) return undefined
 317
 318   const categoryString = categories[0]
 319   if (!categoryString || typeof categoryString !== 'string') return undefined
 320
 321   if (categoryString === 'News & Politics') return 11
 322
 323   for (const key of Object.keys(VIDEO_CATEGORIES)) {
 324     const category = VIDEO_CATEGORIES[key]
 325     if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
 326   }
 327
 328   return undefined
 329 }
 330
 331 function getLanguage (language: string) {
 332   return VIDEO_LANGUAGES[language] ? language : undefined
 333 }
 334
 335 function wrapWithProxyOptions (options: string[]) {
 336   if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
 337     logger.debug('Using proxy for YoutubeDL')
 338
 339     return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options)
 340   }
 341
 342   return options
 343 }