server/tools/peertube-import-videos.ts

   1 // FIXME: https://github.com/nodejs/node/pull/16853
   2 require('tls').DEFAULT_ECDH_CURVE = 'auto'
   3
   4 import * as program from 'commander'
   5 import { join } from 'path'
   6 import { VideoPrivacy } from '../../shared/models/videos'
   7 import { doRequestAndSaveToFile } from '../helpers/requests'
   8 import { CONSTRAINTS_FIELDS } from '../initializers/constants'
   9 import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../../shared/extra-utils/index'
  10 import { truncate } from 'lodash'
  11 import * as prompt from 'prompt'
  12 import { remove } from 'fs-extra'
  13 import { sha256 } from '../helpers/core-utils'
  14 import { buildOriginallyPublishedAt, safeGetYoutubeDL } from '../helpers/youtube-dl'
  15 import { getNetrc, getRemoteObjectOrDie, getSettings } from './cli'
  16
  17 let accessToken: string
  18 let client: { id: string, secret: string }
  19
  20 const processOptions = {
  21   cwd: __dirname,
  22   maxBuffer: Infinity
  23 }
  24
  25 program
  26   .name('import-videos')
  27   .option('-u, --url <url>', 'Server url')
  28   .option('-U, --username <username>', 'Username')
  29   .option('-p, --password <token>', 'Password')
  30   .option('-t, --target-url <targetUrl>', 'Video target URL')
  31   .option('-l, --language <languageCode>', 'Language ISO 639 code (fr or en...)')
  32   .option('-v, --verbose', 'Verbose mode')
  33   .parse(process.argv)
  34
  35 Promise.all([ getSettings(), getNetrc() ])
  36        .then(([ settings, netrc ]) => {
  37          const { url, username, password } = getRemoteObjectOrDie(program, settings)
  38
  39          if (!program[ 'targetUrl' ]) {
  40            console.error('--targetUrl field is required.')
  41
  42            process.exit(-1)
  43          }
  44
  45          removeEndSlashes(url)
  46          removeEndSlashes(program[ 'targetUrl' ])
  47
  48          const user = {
  49            username: username,
  50            password: password
  51          }
  52
  53          run(user, url)
  54            .catch(err => {
  55              console.error(err)
  56              process.exit(-1)
  57            })
  58        })
  59
  60 async function promptPassword () {
  61   return new Promise((res, rej) => {
  62     prompt.start()
  63     const schema = {
  64       properties: {
  65         password: {
  66           hidden: true,
  67           required: true
  68         }
  69       }
  70     }
  71     prompt.get(schema, function (err, result) {
  72       if (err) {
  73         return rej(err)
  74       }
  75       return res(result.password)
  76     })
  77   })
  78 }
  79
  80 async function run (user, url: string) {
  81   if (!user.password) {
  82     user.password = await promptPassword()
  83   }
  84
  85   const res = await getClient(url)
  86   client = {
  87     id: res.body.client_id,
  88     secret: res.body.client_secret
  89   }
  90
  91   try {
  92     const res = await login(program[ 'url' ], client, user)
  93     accessToken = res.body.access_token
  94   } catch (err) {
  95     throw new Error('Cannot authenticate. Please check your username/password.')
  96   }
  97
  98   const youtubeDL = await safeGetYoutubeDL()
  99
 100   const options = [ '-j', '--flat-playlist', '--playlist-reverse' ]
 101   youtubeDL.getInfo(program[ 'targetUrl' ], options, processOptions, async (err, info) => {
 102     if (err) {
 103       console.log(err.message)
 104       process.exit(1)
 105     }
 106
 107     let infoArray: any[]
 108
 109     // Normalize utf8 fields
 110     if (Array.isArray(info) === true) {
 111       infoArray = info.map(i => normalizeObject(i))
 112     } else {
 113       infoArray = [ normalizeObject(info) ]
 114     }
 115     console.log('Will download and upload %d videos.\n', infoArray.length)
 116
 117     for (const info of infoArray) {
 118       await processVideo(info, program[ 'language' ], processOptions.cwd, url, user)
 119     }
 120
 121     console.log('Video/s for user %s imported: %s', program[ 'username' ], program[ 'targetUrl' ])
 122     process.exit(0)
 123   })
 124 }
 125
 126 function processVideo (info: any, languageCode: string, cwd: string, url: string, user) {
 127   return new Promise(async res => {
 128     if (program[ 'verbose' ]) console.log('Fetching object.', info)
 129
 130     const videoInfo = await fetchObject(info)
 131     if (program[ 'verbose' ]) console.log('Fetched object.', videoInfo)
 132
 133     const result = await searchVideoWithSort(url, videoInfo.title, '-match')
 134
 135     console.log('############################################################\n')
 136
 137     if (result.body.data.find(v => v.name === videoInfo.title)) {
 138       console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
 139       return res()
 140     }
 141
 142     const path = join(cwd, sha256(videoInfo.url) + '.mp4')
 143
 144     console.log('Downloading video "%s"...', videoInfo.title)
 145
 146     const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
 147     try {
 148       const youtubeDL = await safeGetYoutubeDL()
 149       youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
 150         if (err) {
 151           console.error(err)
 152           return res()
 153         }
 154
 155         console.log(output.join('\n'))
 156         await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, cwd, url, user, languageCode)
 157         return res()
 158       })
 159     } catch (err) {
 160       console.log(err.message)
 161       return res()
 162     }
 163   })
 164 }
 165
 166 async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: string, url: string, user, language?: string) {
 167   const category = await getCategory(videoInfo.categories, url)
 168   const licence = getLicence(videoInfo.license)
 169   let tags = []
 170   if (Array.isArray(videoInfo.tags)) {
 171     tags = videoInfo.tags
 172                     .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
 173                     .map(t => t.normalize())
 174                     .slice(0, 5)
 175   }
 176
 177   let thumbnailfile
 178   if (videoInfo.thumbnail) {
 179     thumbnailfile = join(cwd, sha256(videoInfo.thumbnail) + '.jpg')
 180
 181     await doRequestAndSaveToFile({
 182       method: 'GET',
 183       uri: videoInfo.thumbnail
 184     }, thumbnailfile)
 185   }
 186
 187   const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo)
 188
 189   const videoAttributes = {
 190     name: truncate(videoInfo.title, {
 191       'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
 192       'separator': /,? +/,
 193       'omission': ' […]'
 194     }),
 195     category,
 196     licence,
 197     language,
 198     nsfw: isNSFW(videoInfo),
 199     waitTranscoding: true,
 200     commentsEnabled: true,
 201     downloadEnabled: true,
 202     description: videoInfo.description || undefined,
 203     support: undefined,
 204     tags,
 205     privacy: VideoPrivacy.PUBLIC,
 206     fixture: videoPath,
 207     thumbnailfile,
 208     previewfile: thumbnailfile,
 209     originallyPublishedAt: originallyPublishedAt ? originallyPublishedAt.toISOString() : null
 210   }
 211
 212   console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
 213   try {
 214     await uploadVideo(url, accessToken, videoAttributes)
 215   } catch (err) {
 216     if (err.message.indexOf('401') !== -1) {
 217       console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.')
 218
 219       const res = await login(url, client, user)
 220       accessToken = res.body.access_token
 221
 222       await uploadVideo(url, accessToken, videoAttributes)
 223     } else {
 224       console.log(err.message)
 225       process.exit(1)
 226     }
 227   }
 228
 229   await remove(videoPath)
 230   if (thumbnailfile) await remove(thumbnailfile)
 231
 232   console.log('Uploaded video "%s"!\n', videoAttributes.name)
 233 }
 234
 235 async function getCategory (categories: string[], url: string) {
 236   if (!categories) return undefined
 237
 238   const categoryString = categories[ 0 ]
 239
 240   if (categoryString === 'News & Politics') return 11
 241
 242   const res = await getVideoCategories(url)
 243   const categoriesServer = res.body
 244
 245   for (const key of Object.keys(categoriesServer)) {
 246     const categoryServer = categoriesServer[ key ]
 247     if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10)
 248   }
 249
 250   return undefined
 251 }
 252
 253 /* ---------------------------------------------------------- */
 254
 255 function getLicence (licence: string) {
 256   if (!licence) return undefined
 257
 258   if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
 259
 260   return undefined
 261 }
 262
 263 function normalizeObject (obj: any) {
 264   const newObj: any = {}
 265
 266   for (const key of Object.keys(obj)) {
 267     // Deprecated key
 268     if (key === 'resolution') continue
 269
 270     const value = obj[ key ]
 271
 272     if (typeof value === 'string') {
 273       newObj[ key ] = value.normalize()
 274     } else {
 275       newObj[ key ] = value
 276     }
 277   }
 278
 279   return newObj
 280 }
 281
 282 function fetchObject (info: any) {
 283   const url = buildUrl(info)
 284
 285   return new Promise<any>(async (res, rej) => {
 286     const youtubeDL = await safeGetYoutubeDL()
 287     youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => {
 288       if (err) return rej(err)
 289
 290       const videoInfoWithUrl = Object.assign(videoInfo, { url })
 291       return res(normalizeObject(videoInfoWithUrl))
 292     })
 293   })
 294 }
 295
 296 function buildUrl (info: any) {
 297   const webpageUrl = info.webpage_url as string
 298   if (webpageUrl && webpageUrl.match(/^https?:\/\//)) return webpageUrl
 299
 300   const url = info.url as string
 301   if (url && url.match(/^https?:\/\//)) return url
 302
 303   // It seems youtube-dl does not return the video url
 304   return 'https://www.youtube.com/watch?v=' + info.id
 305 }
 306
 307 function isNSFW (info: any) {
 308   if (info.age_limit && info.age_limit >= 16) return true
 309
 310   return false
 311 }
 312
 313 function removeEndSlashes (url: string) {
 314   while (url.endsWith('/')) {
 315     url.slice(0, -1)
 316   }
 317 }