server/helpers/youtube-dl.ts

   1 import { truncate } from 'lodash'
   2 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
   3 import { logger } from './logger'
   4 import { generateVideoImportTmpPath } from './utils'
   5 import { join } from 'path'
   6 import { root } from './core-utils'
   7 import { ensureDir, writeFile, remove } from 'fs-extra'
   8 import * as request from 'request'
   9 import { createWriteStream } from 'fs'
  10
  11 export type YoutubeDLInfo = {
  12   name?: string
  13   description?: string
  14   category?: number
  15   licence?: number
  16   nsfw?: boolean
  17   tags?: string[]
  18   thumbnailUrl?: string
  19   originallyPublishedAt?: string
  20 }
  21
  22 const processOptions = {
  23   maxBuffer: 1024 * 1024 * 10 // 10MB
  24 }
  25
  26 function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
  27   return new Promise<YoutubeDLInfo>(async (res, rej) => {
  28     const args = opts || [ '-j', '--flat-playlist' ]
  29
  30     const youtubeDL = await safeGetYoutubeDL()
  31     youtubeDL.getInfo(url, args, processOptions, (err, info) => {
  32       if (err) return rej(err)
  33       if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
  34
  35       const obj = buildVideoInfo(normalizeObject(info))
  36       if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
  37
  38       return res(obj)
  39     })
  40   })
  41 }
  42
  43 function downloadYoutubeDLVideo (url: string, timeout: number) {
  44   const path = generateVideoImportTmpPath(url)
  45   let timer
  46
  47   logger.info('Importing youtubeDL video %s', url)
  48
  49   const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
  50
  51   return new Promise<string>(async (res, rej) => {
  52     const youtubeDL = await safeGetYoutubeDL()
  53     youtubeDL.exec(url, options, processOptions, err => {
  54       clearTimeout(timer)
  55
  56       if (err) {
  57         remove(path)
  58           .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
  59
  60         return rej(err)
  61       }
  62
  63       return res(path)
  64     })
  65
  66     timer = setTimeout(async () => {
  67       await remove(path)
  68
  69       return rej(new Error('YoutubeDL download timeout.'))
  70     }, timeout)
  71   })
  72 }
  73
  74 // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
  75 // We rewrote it to avoid sync calls
  76 async function updateYoutubeDLBinary () {
  77   logger.info('Updating youtubeDL binary.')
  78
  79   const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
  80   const bin = join(binDirectory, 'youtube-dl')
  81   const detailsPath = join(binDirectory, 'details')
  82   const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
  83
  84   await ensureDir(binDirectory)
  85
  86   return new Promise(res => {
  87     request.get(url, { followRedirect: false }, (err, result) => {
  88       if (err) {
  89         logger.error('Cannot update youtube-dl.', { err })
  90         return res()
  91       }
  92
  93       if (result.statusCode !== 302) {
  94         logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
  95         return res()
  96       }
  97
  98       const url = result.headers.location
  99       const downloadFile = request.get(url)
 100       const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
 101
 102       downloadFile.on('response', result => {
 103         if (result.statusCode !== 200) {
 104           logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
 105           return res()
 106         }
 107
 108         downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
 109       })
 110
 111       downloadFile.on('error', err => {
 112         logger.error('youtube-dl update error.', { err })
 113         return res()
 114       })
 115
 116       downloadFile.on('end', () => {
 117         const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
 118         writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
 119           if (err) {
 120             logger.error('youtube-dl update error: cannot write details.', { err })
 121             return res()
 122           }
 123
 124           logger.info('youtube-dl updated to version %s.', newVersion)
 125           return res()
 126         })
 127       })
 128     })
 129   })
 130 }
 131
 132 async function safeGetYoutubeDL () {
 133   let youtubeDL
 134
 135   try {
 136     youtubeDL = require('youtube-dl')
 137   } catch (e) {
 138     // Download binary
 139     await updateYoutubeDLBinary()
 140     youtubeDL = require('youtube-dl')
 141   }
 142
 143   return youtubeDL
 144 }
 145
 146 // ---------------------------------------------------------------------------
 147
 148 export {
 149   updateYoutubeDLBinary,
 150   downloadYoutubeDLVideo,
 151   getYoutubeDLInfo,
 152   safeGetYoutubeDL
 153 }
 154
 155 // ---------------------------------------------------------------------------
 156
 157 function normalizeObject (obj: any) {
 158   const newObj: any = {}
 159
 160   for (const key of Object.keys(obj)) {
 161     // Deprecated key
 162     if (key === 'resolution') continue
 163
 164     const value = obj[key]
 165
 166     if (typeof value === 'string') {
 167       newObj[key] = value.normalize()
 168     } else {
 169       newObj[key] = value
 170     }
 171   }
 172
 173   return newObj
 174 }
 175
 176 function buildVideoInfo (obj: any) {
 177
 178   const date = obj.upload_date.slice(0,4) + ',' + obj.upload_date.slice(4,6) + ',' + obj.upload_date.slice(6,8)
 179
 180   return {
 181     name: titleTruncation(obj.title),
 182     description: descriptionTruncation(obj.description),
 183     category: getCategory(obj.categories),
 184     licence: getLicence(obj.license),
 185     nsfw: isNSFW(obj),
 186     tags: getTags(obj.tags),
 187     thumbnailUrl: obj.thumbnail || undefined,
 188     originallyPublishedAt: new Date(date).toISOString()
 189   }
 190 }
 191
 192 function titleTruncation (title: string) {
 193   return truncate(title, {
 194     'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
 195     'separator': /,? +/,
 196     'omission': ' […]'
 197   })
 198 }
 199
 200 function descriptionTruncation (description: string) {
 201   if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
 202
 203   return truncate(description, {
 204     'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
 205     'separator': /,? +/,
 206     'omission': ' […]'
 207   })
 208 }
 209
 210 function isNSFW (info: any) {
 211   return info.age_limit && info.age_limit >= 16
 212 }
 213
 214 function getTags (tags: any) {
 215   if (Array.isArray(tags) === false) return []
 216
 217   return tags
 218     .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
 219     .map(t => t.normalize())
 220     .slice(0, 5)
 221 }
 222
 223 function getLicence (licence: string) {
 224   if (!licence) return undefined
 225
 226   if (licence.indexOf('Creative Commons Attribution') !== -1) return 1
 227
 228   return undefined
 229 }
 230
 231 function getCategory (categories: string[]) {
 232   if (!categories) return undefined
 233
 234   const categoryString = categories[0]
 235   if (!categoryString || typeof categoryString !== 'string') return undefined
 236
 237   if (categoryString === 'News & Politics') return 11
 238
 239   for (const key of Object.keys(VIDEO_CATEGORIES)) {
 240     const category = VIDEO_CATEGORIES[key]
 241     if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
 242   }
 243
 244   return undefined
 245 }