stricter youtubedl format selectors (#3516)

[github/Chocobozzz/PeerTube.git] / server / tools / peertube-import-videos.ts

diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
index 0efe87810c98f4d2bc39a40bf548b63c67e67b8c..3a82b3832110c46870503e45c559986a009e8f58 100644 (file)
--- a/server/tools/peertube-import-videos.ts
+++ b/server/tools/peertube-import-videos.ts
@@ -11,7 +11,7 @@ import * as prompt from 'prompt'
 import { accessSync, constants } from 'fs'
 import { remove } from 'fs-extra'
 import { sha256 } from '../helpers/core-utils'
-import { buildOriginallyPublishedAt, safeGetYoutubeDL } from '../helpers/youtube-dl'
+import { buildOriginallyPublishedAt, getYoutubeDLVideoFormat, safeGetYoutubeDL } from '../helpers/youtube-dl'
 import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli'
 
 type UserInfo = {
@@ -38,6 +38,7 @@ command
   .option('--first <first>', 'Process first n elements of returned playlist')
   .option('--last <last>', 'Process last n elements of returned playlist')
   .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname)
+  .usage("[global options] [ -- youtube-dl options]")
   .parse(process.argv)
 
 const log = getLogger(program['verbose'])
@@ -71,37 +72,47 @@ async function run (url: string, user: UserInfo) {
 
   const youtubeDL = await safeGetYoutubeDL()
 
-  const options = [ '-j', '--flat-playlist', '--playlist-reverse' ]
-  youtubeDL.getInfo(program['targetUrl'], options, processOptions, async (err, info) => {
-    if (err) {
-      exitError(err.message)
-    }
+  let info = await getYoutubeDLInfo(youtubeDL, program['targetUrl'], command.args)
 
-    let infoArray: any[]
+  if (!Array.isArray(info)) info = [ info ]
 
-    // Normalize utf8 fields
-    infoArray = [].concat(info)
-    if (program['first']) {
-      infoArray = infoArray.slice(0, program['first'])
-    } else if (program['last']) {
-      infoArray = infoArray.slice(-program['last'])
-    }
-    infoArray = infoArray.map(i => normalizeObject(i))
+  // Try to fix youtube channels upload
+  const uploadsObject = info.find(i => !i.ie_key && !i.duration && i.title === 'Uploads')
+
+  if (uploadsObject) {
+    console.log('Fixing URL to %s.', uploadsObject.url)
+
+    info = await getYoutubeDLInfo(youtubeDL, uploadsObject.url, command.args)
+  }
+
+  let infoArray: any[]
 
-    log.info('Will download and upload %d videos.\n', infoArray.length)
+  // Normalize utf8 fields
+  infoArray = [].concat(info)
+  if (program['first']) {
+    infoArray = infoArray.slice(0, program['first'])
+  } else if (program['last']) {
+    infoArray = infoArray.slice(-program['last'])
+  }
+  infoArray = infoArray.map(i => normalizeObject(i))
+
+  log.info('Will download and upload %d videos.\n', infoArray.length)
 
-    for (const info of infoArray) {
+  for (const info of infoArray) {
+    try {
       await processVideo({
         cwd: program['tmpdir'],
         url,
         user,
         youtubeInfo: info
       })
+    } catch (err) {
+      console.error('Cannot process video.', { info, url })
     }
+  }
 
-    log.info('Video/s for user %s imported: %s', user.username, program['targetUrl'])
-    process.exit(0)
-  })
+  log.info('Video/s for user %s imported: %s', user.username, program['targetUrl'])
+  process.exit(0)
 }
 
 function processVideo (parameters: {
@@ -118,19 +129,18 @@ function processVideo (parameters: {
     const videoInfo = await fetchObject(youtubeInfo)
     log.debug('Fetched object.', videoInfo)
 
-    if (program['since']) {
-      if (buildOriginallyPublishedAt(videoInfo).getTime() < program['since'].getTime()) {
-        log.info('Video "%s" has been published before "%s", don\'t upload it.\n',
-          videoInfo.title, formatDate(program['since']))
-        return res()
-      }
+    const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo)
+
+    if (program['since'] && originallyPublishedAt && originallyPublishedAt.getTime() < program['since'].getTime()) {
+      log.info('Video "%s" has been published before "%s", don\'t upload it.\n',
+        videoInfo.title, formatDate(program['since']))
+      return res()
     }
-    if (program['until']) {
-      if (buildOriginallyPublishedAt(videoInfo).getTime() > program['until'].getTime()) {
-        log.info('Video "%s" has been published after "%s", don\'t upload it.\n',
-          videoInfo.title, formatDate(program['until']))
-        return res()
-      }
+
+    if (program['until'] && originallyPublishedAt && originallyPublishedAt.getTime() > program['until'].getTime()) {
+      log.info('Video "%s" has been published after "%s", don\'t upload it.\n',
+        videoInfo.title, formatDate(program['until']))
+      return res()
     }
 
     const result = await searchVideoWithSort(url, videoInfo.title, '-match')
@@ -146,7 +156,7 @@ function processVideo (parameters: {
 
     log.info('Downloading video "%s"...', videoInfo.title)
 
-    const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
+    const options = [ '-f', getYoutubeDLVideoFormat(), ...command.args, '-o', path ]
     try {
       const youtubeDL = await safeGetYoutubeDL()
       youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
@@ -272,7 +282,7 @@ async function getCategory (categories: string[], url: string) {
 function getLicence (licence: string) {
   if (!licence) return undefined
 
-  if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
+  if (licence.includes('Creative Commons Attribution licence')) return 1
 
   return undefined
 }
@@ -391,3 +401,15 @@ function exitError (message: string, ...meta: any[]) {
   console.error(message, ...meta)
   process.exit(-1)
 }
+
+function getYoutubeDLInfo (youtubeDL: any, url: string, args: string[]) {
+  return new Promise<any>((res, rej) => {
+    const options = [ '-j', '--flat-playlist', '--playlist-reverse', ...args ]
+
+    youtubeDL.getInfo(url, options, processOptions, async (err, info) => {
+      if (err) return rej(err)
+
+      return res(info)
+    })
+  })
+}