Fix remote image fetching

[github/Chocobozzz/PeerTube.git] / server / tools / import-youtube.ts

diff --git a/server/tools/import-youtube.ts b/server/tools/import-youtube.ts
index b4405c4522830e00731b1bfc60f3030df6e35e14..ab91afbc060a263932bce94d6f6109a966da0679 100644 (file)
--- a/server/tools/import-youtube.ts
+++ b/server/tools/import-youtube.ts
@@ -1,17 +1,18 @@
 import * as program from 'commander'
-import { createWriteStream } from 'fs'
 import { join } from 'path'
-import { cursorTo } from 'readline'
 import * as youtubeDL from 'youtube-dl'
 import { VideoPrivacy } from '../../shared/models/videos'
 import { unlinkPromise } from '../helpers/core-utils'
+import { doRequestAndSaveToFile } from '../helpers/requests'
+import { CONSTRAINTS_FIELDS } from '../initializers'
 import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils'
 
 program
   .option('-u, --url <url>', 'Server url')
   .option('-U, --username <username>', 'Username')
   .option('-p, --password <token>', 'Password')
-  .option('-y, --youtube-url <directory>', 'Youtube URL')
+  .option('-y, --youtube-url <youtubeUrl>', 'Youtube URL')
+  .option('-l, --language <languageCode>', 'Language code')
   .parse(process.argv)
 
 if (
@@ -20,12 +21,17 @@ if (
   !program['password'] ||
   !program['youtubeUrl']
 ) {
-  throw new Error('All arguments are required.')
+  console.error('All arguments are required.')
+  process.exit(-1)
 }
 
 run().catch(err => console.error(err))
 
 let accessToken: string
+const processOptions = {
+  cwd: __dirname,
+  maxBuffer: Infinity
+}
 
 async function run () {
   const res = await getClient(program['url'])
@@ -42,9 +48,13 @@ async function run () {
   const res2 = await login(program['url'], client, user)
   accessToken = res2.body.access_token
 
-  youtubeDL.getInfo(program['youtubeUrl'], [ '-j', '--flat-playlist' ], async (err, info) => {
+  const options = [ '-j', '--flat-playlist' ]
+  youtubeDL.getInfo(program['youtubeUrl'], options, processOptions, async (err, info) => {
     if (err) throw err
 
+    // Normalize utf8 fields
+    info = info.map(i => normalizeObject(i))
+
     const videos = info.map(i => {
       return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title }
     })
@@ -52,7 +62,7 @@ async function run () {
     console.log('Will download and upload %d videos.\n', videos.length)
 
     for (const video of videos) {
-      await processVideo(video)
+      await processVideo(video, program['language'])
     }
 
     console.log('I\'m finished!')
@@ -60,58 +70,58 @@ async function run () {
   })
 }
 
-function processVideo (videoUrlName: { name: string, url: string }) {
+function processVideo (video: { name: string, url: string }, languageCode: number) {
   return new Promise(async res => {
-    const result = await searchVideo(program['url'], videoUrlName.name)
+    const result = await searchVideo(program['url'], video.name)
+
+    console.log('############################################################\n')
+
     if (result.body.total !== 0) {
-      console.log('Video "%s" already exist, don\'t reupload it.\n', videoUrlName.name)
+      console.log('Video "%s" already exists, don\'t reupload it.\n', video.name)
       return res()
     }
 
-    const video = youtubeDL(videoUrlName.url)
-    let videoInfo
-    let videoPath: string
+    const path = join(__dirname, new Date().getTime() + '.mp4')
 
-    video.on('error', err => console.error(err))
+    console.log('Downloading video "%s"...', video.name)
 
-    let size = 0
-    video.on('info', info => {
-      videoInfo = info
-      size = info.size
+    const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ]
+    youtubeDL.exec(video.url, options, processOptions, async (err, output) => {
+      if (err) return console.error(err)
 
-      videoPath = join(__dirname, size + '.mp4')
-      console.log('Creating "%s" of video "%s".', videoPath, videoInfo.title)
+      console.log(output.join('\n'))
 
-      video.pipe(createWriteStream(videoPath))
-    })
-
-    let pos = 0
-    video.on('data', chunk => {
-      pos += chunk.length
-      // `size` should not be 0 here.
-      if (size) {
-        const percent = (pos / size * 100).toFixed(2)
-        writeWaitingPercent(percent)
-      }
-    })
+      youtubeDL.getInfo(video.url, undefined, processOptions, async (err, videoInfo) => {
+        if (err) return console.error(err)
 
-    video.on('end', async () => {
-      await uploadVideoOnPeerTube(videoInfo, videoPath)
+        await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, languageCode)
 
-      return res()
+        return res()
+      })
     })
   })
 }
 
-function writeWaitingPercent (p: string) {
-  cursorTo(process.stdout, 0)
-  process.stdout.write(`waiting ... ${p}%`)
-}
-
-async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string) {
+async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, language?: number) {
   const category = await getCategory(videoInfo.categories)
   const licence = getLicence(videoInfo.license)
-  const language = 13
+  let tags = []
+  if (Array.isArray(videoInfo.tags)) {
+    tags = videoInfo.tags
+      .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max)
+      .map(t => t.normalize())
+      .slice(0, 5)
+  }
+
+  let thumbnailfile
+  if (videoInfo.thumbnail) {
+    thumbnailfile = join(__dirname, 'thumbnail.jpg')
+
+    await doRequestAndSaveToFile({
+      method: 'GET',
+      uri: videoInfo.thumbnail
+    }, thumbnailfile)
+  }
 
   const videoAttributes = {
     name: videoInfo.title,
@@ -121,14 +131,21 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string) {
     nsfw: false,
     commentsEnabled: true,
     description: videoInfo.description,
-    tags: videoInfo.tags.slice(0, 5),
+    tags,
     privacy: VideoPrivacy.PUBLIC,
-    fixture: videoPath
+    fixture: videoPath,
+    thumbnailfile,
+    previewfile: thumbnailfile
   }
 
   console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
   await uploadVideo(program['url'], accessToken, videoAttributes)
+
   await unlinkPromise(videoPath)
+  if (thumbnailfile) {
+    await unlinkPromise(thumbnailfile)
+  }
+
   console.log('Uploaded video "%s"!\n', videoAttributes.name)
 }
 
@@ -153,3 +170,22 @@ function getLicence (licence: string) {
 
   return undefined
 }
+
+function normalizeObject (obj: any) {
+  const newObj: any = {}
+
+  for (const key of Object.keys(obj)) {
+    // Deprecated key
+    if (key === 'resolution') continue
+
+    const value = obj[key]
+
+    if (typeof value === 'string') {
+      newObj[key] = value.normalize()
+    } else {
+      newObj[key] = value
+    }
+  }
+
+  return newObj
+}