X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fhelpers%2Fyoutube-dl.ts;h=74e5f896c1a49929315c931c4efb6c2ca76a78bd;hb=b3fa96eea0c6842459f978385a6b424a0beee4fa;hp=fc9d416a179cd9f9ba92bab7523ac2a062f4bf59;hpb=a15871560f80e07386c1dabb8370cd2664ecfd1f;p=github%2FChocobozzz%2FPeerTube.git

diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index fc9d416a1..74e5f896c 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,4 +1,4 @@
-import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers/constants'
+import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
 import { logger } from './logger'
 import { generateVideoImportTmpPath } from './utils'
 import { join } from 'path'
@@ -7,18 +7,27 @@ import { ensureDir, remove, writeFile } from 'fs-extra'
 import * as request from 'request'
 import { createWriteStream } from 'fs'
 import { CONFIG } from '@server/initializers/config'
+import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
 
 export type YoutubeDLInfo = {
   name?: string
   description?: string
   category?: number
+  language?: string
   licence?: number
   nsfw?: boolean
   tags?: string[]
   thumbnailUrl?: string
+  fileExt?: string
   originallyPublishedAt?: Date
 }
 
+export type YoutubeDLSubs = {
+  language: string
+  filename: string
+  path: string
+}[]
+
 const processOptions = {
   maxBuffer: 1024 * 1024 * 10 // 10MB
 }
@@ -26,6 +35,11 @@ const processOptions = {
 function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
   return new Promise<YoutubeDLInfo>((res, rej) => {
     let args = opts || [ '-j', '--flat-playlist' ]
+
+    if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
+      args.push('--force-ipv4')
+    }
+
     args = wrapWithProxyOptions(args)
 
     safeGetYoutubeDL()
@@ -44,11 +58,45 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
   })
 }
 
-function downloadYoutubeDLVideo (url: string, timeout: number) {
-  const path = generateVideoImportTmpPath(url)
+function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
+  return new Promise<YoutubeDLSubs>((res, rej) => {
+    const cwd = CONFIG.STORAGE.TMP_DIR
+    const options = opts || { all: true, format: 'vtt', cwd }
+
+    safeGetYoutubeDL()
+      .then(youtubeDL => {
+        youtubeDL.getSubs(url, options, (err, files) => {
+          if (err) return rej(err)
+          if (!files) return []
+
+          logger.debug('Get subtitles from youtube dl.', { url, files })
+
+          const subtitles = files.reduce((acc, filename) => {
+            const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
+            if (!matched || !matched[1]) return acc
+
+            return [
+              ...acc,
+              {
+                language: matched[1],
+                path: join(cwd, filename),
+                filename
+              }
+            ]
+          }, [])
+
+          return res(subtitles)
+        })
+      })
+      .catch(err => rej(err))
+  })
+}
+
+function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
+  const path = generateVideoImportTmpPath(url, extension)
   let timer
 
-  logger.info('Importing youtubeDL video %s', url)
+  logger.info('Importing youtubeDL video %s to %s', url, path)
 
   let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
   options = wrapWithProxyOptions(options)
@@ -96,7 +144,7 @@ async function updateYoutubeDLBinary () {
   const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
   const bin = join(binDirectory, 'youtube-dl')
   const detailsPath = join(binDirectory, 'details')
-  const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
+  const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl'
 
   await ensureDir(binDirectory)
 
@@ -107,7 +155,7 @@ async function updateYoutubeDLBinary () {
         return res()
       }
 
-      if (result.statusCode !== 302) {
+      if (result.statusCode !== HttpStatusCode.FOUND_302) {
         logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
         return res()
       }
@@ -117,12 +165,17 @@ async function updateYoutubeDLBinary () {
       const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
 
       downloadFile.on('response', result => {
-        if (result.statusCode !== 200) {
+        if (result.statusCode !== HttpStatusCode.OK_200) {
           logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
           return res()
         }
 
-        downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
+        const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => {
+          logger.error('youtube-dl update error in write stream', { err })
+          return res()
+        })
+
+        downloadFile.pipe(writeStream)
       })
 
       downloadFile.on('error', err => {
@@ -184,6 +237,7 @@ function buildOriginallyPublishedAt (obj: any) {
 export {
   updateYoutubeDLBinary,
   downloadYoutubeDLVideo,
+  getYoutubeDLSubs,
   getYoutubeDLInfo,
   safeGetYoutubeDL,
   buildOriginallyPublishedAt
@@ -210,16 +264,18 @@ function normalizeObject (obj: any) {
   return newObj
 }
 
-function buildVideoInfo (obj: any) {
+function buildVideoInfo (obj: any): YoutubeDLInfo {
   return {
     name: titleTruncation(obj.title),
     description: descriptionTruncation(obj.description),
     category: getCategory(obj.categories),
     licence: getLicence(obj.license),
+    language: getLanguage(obj.language),
     nsfw: isNSFW(obj),
     tags: getTags(obj.tags),
     thumbnailUrl: obj.thumbnail || undefined,
-    originallyPublishedAt: buildOriginallyPublishedAt(obj)
+    originallyPublishedAt: buildOriginallyPublishedAt(obj),
+    fileExt: obj.ext
   }
 }
 
@@ -257,7 +313,12 @@ function getTags (tags: any) {
 function getLicence (licence: string) {
   if (!licence) return undefined
 
-  if (licence.indexOf('Creative Commons Attribution') !== -1) return 1
+  if (licence.includes('Creative Commons Attribution')) return 1
+
+  for (const key of Object.keys(VIDEO_LICENCES)) {
+    const peertubeLicence = VIDEO_LICENCES[key]
+    if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
+  }
 
   return undefined
 }
@@ -278,6 +339,10 @@ function getCategory (categories: string[]) {
   return undefined
 }
 
+function getLanguage (language: string) {
+  return VIDEO_LANGUAGES[language] ? language : undefined
+}
+
 function wrapWithProxyOptions (options: string[]) {
   if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
     logger.debug('Using proxy for YoutubeDL')