server/tools/peertube-import-videos.ts

   1 // FIXME: https://github.com/nodejs/node/pull/16853
   2 require('tls').DEFAULT_ECDH_CURVE = 'auto'
   3
   4 import * as program from 'commander'
   5 import { join } from 'path'
   6 import { doRequestAndSaveToFile } from '../helpers/requests'
   7 import { CONSTRAINTS_FIELDS } from '../initializers/constants'
   8 import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../../shared/extra-utils/index'
   9 import { truncate } from 'lodash'
  10 import * as prompt from 'prompt'
  11 import { remove } from 'fs-extra'
  12 import { sha256 } from '../helpers/core-utils'
  13 import { buildOriginallyPublishedAt, safeGetYoutubeDL } from '../helpers/youtube-dl'
  14 import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getNetrc, getRemoteObjectOrDie, getSettings } from './cli'
  15
  16 type UserInfo = {
  17   username: string
  18   password: string
  19 }
  20
  21 const processOptions = {
  22   cwd: __dirname,
  23   maxBuffer: Infinity
  24 }
  25
  26 let command = program
  27   .name('import-videos')
  28
  29 command = buildCommonVideoOptions(command)
  30
  31 command
  32   .option('-u, --url <url>', 'Server url')
  33   .option('-U, --username <username>', 'Username')
  34   .option('-p, --password <token>', 'Password')
  35   .option('-t, --target-url <targetUrl>', 'Video target URL')
  36   .option('-v, --verbose', 'Verbose mode')
  37   .parse(process.argv)
  38
  39 Promise.all([ getSettings(), getNetrc() ])
  40        .then(([ settings, netrc ]) => {
  41          const { url, username, password } = getRemoteObjectOrDie(program, settings, netrc)
  42
  43          if (!program[ 'targetUrl' ]) {
  44            console.error('--targetUrl field is required.')
  45
  46            process.exit(-1)
  47          }
  48
  49          removeEndSlashes(url)
  50          removeEndSlashes(program[ 'targetUrl' ])
  51
  52          const user = { username, password }
  53
  54          run(url, user)
  55            .catch(err => {
  56              console.error(err)
  57              process.exit(-1)
  58            })
  59        })
  60
  61 async function run (url: string, user: UserInfo) {
  62   if (!user.password) {
  63     user.password = await promptPassword()
  64   }
  65
  66   const youtubeDL = await safeGetYoutubeDL()
  67
  68   const options = [ '-j', '--flat-playlist', '--playlist-reverse' ]
  69   youtubeDL.getInfo(program[ 'targetUrl' ], options, processOptions, async (err, info) => {
  70     if (err) {
  71       console.log(err.message)
  72       process.exit(1)
  73     }
  74
  75     let infoArray: any[]
  76
  77     // Normalize utf8 fields
  78     if (Array.isArray(info) === true) {
  79       infoArray = info.map(i => normalizeObject(i))
  80     } else {
  81       infoArray = [ normalizeObject(info) ]
  82     }
  83     console.log('Will download and upload %d videos.\n', infoArray.length)
  84
  85     for (const info of infoArray) {
  86       await processVideo({
  87         cwd: processOptions.cwd,
  88         url,
  89         user,
  90         youtubeInfo: info
  91       })
  92     }
  93
  94     console.log('Video/s for user %s imported: %s', program[ 'username' ], program[ 'targetUrl' ])
  95     process.exit(0)
  96   })
  97 }
  98
  99 function processVideo (parameters: {
 100   cwd: string,
 101   url: string,
 102   user: { username: string, password: string },
 103   youtubeInfo: any
 104 }) {
 105   const { youtubeInfo, cwd, url, user } = parameters
 106
 107   return new Promise(async res => {
 108     if (program[ 'verbose' ]) console.log('Fetching object.', youtubeInfo)
 109
 110     const videoInfo = await fetchObject(youtubeInfo)
 111     if (program[ 'verbose' ]) console.log('Fetched object.', videoInfo)
 112
 113     const result = await searchVideoWithSort(url, videoInfo.title, '-match')
 114
 115     console.log('############################################################\n')
 116
 117     if (result.body.data.find(v => v.name === videoInfo.title)) {
 118       console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
 119       return res()
 120     }
 121
 122     const path = join(cwd, sha256(videoInfo.url) + '.mp4')
 123
 124     console.log('Downloading video "%s"...', videoInfo.title)
 125
 126     const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
 127     try {
 128       const youtubeDL = await safeGetYoutubeDL()
 129       youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
 130         if (err) {
 131           console.error(err)
 132           return res()
 133         }
 134
 135         console.log(output.join('\n'))
 136         await uploadVideoOnPeerTube({
 137           cwd,
 138           url,
 139           user,
 140           videoInfo: normalizeObject(videoInfo),
 141           videoPath: path
 142         })
 143         return res()
 144       })
 145     } catch (err) {
 146       console.log(err.message)
 147       return res()
 148     }
 149   })
 150 }
 151
 152 async function uploadVideoOnPeerTube (parameters: {
 153   videoInfo: any,
 154   videoPath: string,
 155   cwd: string,
 156   url: string,
 157   user: { username: string; password: string }
 158 }) {
 159   const { videoInfo, videoPath, cwd, url, user } = parameters
 160
 161   const category = await getCategory(videoInfo.categories, url)
 162   const licence = getLicence(videoInfo.license)
 163   let tags = []
 164   if (Array.isArray(videoInfo.tags)) {
 165     tags = videoInfo.tags
 166                     .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
 167                     .map(t => t.normalize())
 168                     .slice(0, 5)
 169   }
 170
 171   let thumbnailfile
 172   if (videoInfo.thumbnail) {
 173     thumbnailfile = join(cwd, sha256(videoInfo.thumbnail) + '.jpg')
 174
 175     await doRequestAndSaveToFile({
 176       method: 'GET',
 177       uri: videoInfo.thumbnail
 178     }, thumbnailfile)
 179   }
 180
 181   const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo)
 182
 183   const defaultAttributes = {
 184     name: truncate(videoInfo.title, {
 185       'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
 186       'separator': /,? +/,
 187       'omission': ' […]'
 188     }),
 189     category,
 190     licence,
 191     nsfw: isNSFW(videoInfo),
 192     description: videoInfo.description,
 193     tags
 194   }
 195
 196   const videoAttributes = await buildVideoAttributesFromCommander(url, program, defaultAttributes)
 197
 198   Object.assign(videoAttributes, {
 199     originallyPublishedAt: originallyPublishedAt ? originallyPublishedAt.toISOString() : null,
 200     thumbnailfile,
 201     previewfile: thumbnailfile,
 202     fixture: videoPath
 203   })
 204
 205   console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
 206
 207   let accessToken = await getAccessTokenOrDie(url, user)
 208
 209   try {
 210     await uploadVideo(url, accessToken, videoAttributes)
 211   } catch (err) {
 212     if (err.message.indexOf('401') !== -1) {
 213       console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.')
 214
 215       accessToken = await getAccessTokenOrDie(url, user)
 216
 217       await uploadVideo(url, accessToken, videoAttributes)
 218     } else {
 219       console.log(err.message)
 220       process.exit(1)
 221     }
 222   }
 223
 224   await remove(videoPath)
 225   if (thumbnailfile) await remove(thumbnailfile)
 226
 227   console.log('Uploaded video "%s"!\n', videoAttributes.name)
 228 }
 229
 230 /* ---------------------------------------------------------- */
 231
 232 async function getCategory (categories: string[], url: string) {
 233   if (!categories) return undefined
 234
 235   const categoryString = categories[ 0 ]
 236
 237   if (categoryString === 'News & Politics') return 11
 238
 239   const res = await getVideoCategories(url)
 240   const categoriesServer = res.body
 241
 242   for (const key of Object.keys(categoriesServer)) {
 243     const categoryServer = categoriesServer[ key ]
 244     if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10)
 245   }
 246
 247   return undefined
 248 }
 249
 250 function getLicence (licence: string) {
 251   if (!licence) return undefined
 252
 253   if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
 254
 255   return undefined
 256 }
 257
 258 function normalizeObject (obj: any) {
 259   const newObj: any = {}
 260
 261   for (const key of Object.keys(obj)) {
 262     // Deprecated key
 263     if (key === 'resolution') continue
 264
 265     const value = obj[ key ]
 266
 267     if (typeof value === 'string') {
 268       newObj[ key ] = value.normalize()
 269     } else {
 270       newObj[ key ] = value
 271     }
 272   }
 273
 274   return newObj
 275 }
 276
 277 function fetchObject (info: any) {
 278   const url = buildUrl(info)
 279
 280   return new Promise<any>(async (res, rej) => {
 281     const youtubeDL = await safeGetYoutubeDL()
 282     youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => {
 283       if (err) return rej(err)
 284
 285       const videoInfoWithUrl = Object.assign(videoInfo, { url })
 286       return res(normalizeObject(videoInfoWithUrl))
 287     })
 288   })
 289 }
 290
 291 function buildUrl (info: any) {
 292   const webpageUrl = info.webpage_url as string
 293   if (webpageUrl && webpageUrl.match(/^https?:\/\//)) return webpageUrl
 294
 295   const url = info.url as string
 296   if (url && url.match(/^https?:\/\//)) return url
 297
 298   // It seems youtube-dl does not return the video url
 299   return 'https://www.youtube.com/watch?v=' + info.id
 300 }
 301
 302 function isNSFW (info: any) {
 303   return info.age_limit && info.age_limit >= 16
 304 }
 305
 306 function removeEndSlashes (url: string) {
 307   while (url.endsWith('/')) {
 308     url.slice(0, -1)
 309   }
 310 }
 311
 312 async function promptPassword () {
 313   return new Promise<string>((res, rej) => {
 314     prompt.start()
 315     const schema = {
 316       properties: {
 317         password: {
 318           hidden: true,
 319           required: true
 320         }
 321       }
 322     }
 323     prompt.get(schema, function (err, result) {
 324       if (err) {
 325         return rej(err)
 326       }
 327       return res(result.password)
 328     })
 329   })
 330 }
 331
 332 async function getAccessTokenOrDie (url: string, user: UserInfo) {
 333   const resClient = await getClient(url)
 334   const client = {
 335     id: resClient.body.client_id,
 336     secret: resClient.body.client_secret
 337   }
 338
 339   try {
 340     const res = await login(url, client, user)
 341     return res.body.access_token
 342   } catch (err) {
 343     console.error('Cannot authenticate. Please check your username/password.')
 344     process.exit(-1)
 345   }
 346 }