server/tools/peertube-import-videos.ts

   1 import { program } from 'commander'
   2 import { accessSync, constants } from 'fs'
   3 import { remove } from 'fs-extra'
   4 import { join } from 'path'
   5 import { YoutubeDLCLI, YoutubeDLInfo, YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl'
   6 import { wait } from '@shared/core-utils'
   7 import { sha256 } from '@shared/extra-utils'
   8 import { doRequestAndSaveToFile } from '../helpers/requests'
   9 import {
  10   assignToken,
  11   buildCommonVideoOptions,
  12   buildServer,
  13   buildVideoAttributesFromCommander,
  14   getLogger,
  15   getServerCredentials
  16 } from './shared'
  17
  18 import prompt = require('prompt')
  19
  20 const processOptions = {
  21   maxBuffer: Infinity
  22 }
  23
  24 let command = program
  25   .name('import-videos')
  26
  27 command = buildCommonVideoOptions(command)
  28
  29 command
  30   .option('-u, --url <url>', 'Server url')
  31   .option('-U, --username <username>', 'Username')
  32   .option('-p, --password <token>', 'Password')
  33   .option('--target-url <targetUrl>', 'Video target URL')
  34   .option('--since <since>', 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)', parseDate)
  35   .option('--until <until>', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate)
  36   .option('--first <first>', 'Process first n elements of returned playlist')
  37   .option('--last <last>', 'Process last n elements of returned playlist')
  38   .option('--wait-interval <waitInterval>', 'Duration between two video imports (in seconds)', convertIntoMs)
  39   .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname)
  40   .usage('[global options] [ -- youtube-dl options]')
  41   .parse(process.argv)
  42
  43 const options = command.opts()
  44
  45 const log = getLogger(options.verbose)
  46
  47 getServerCredentials(command)
  48   .then(({ url, username, password }) => {
  49     if (!options.targetUrl) {
  50       exitError('--target-url field is required.')
  51     }
  52
  53     try {
  54       accessSync(options.tmpdir, constants.R_OK | constants.W_OK)
  55     } catch (e) {
  56       exitError('--tmpdir %s: directory does not exist or is not accessible', options.tmpdir)
  57     }
  58
  59     url = normalizeTargetUrl(url)
  60     options.targetUrl = normalizeTargetUrl(options.targetUrl)
  61
  62     run(url, username, password)
  63       .catch(err => exitError(err))
  64   })
  65   .catch(err => console.error(err))
  66
  67 async function run (url: string, username: string, password: string) {
  68   if (!password) password = await promptPassword()
  69
  70   const youtubeDLBinary = await YoutubeDLCLI.safeGet()
  71
  72   let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args)
  73
  74   if (!Array.isArray(info)) info = [ info ]
  75
  76   // Try to fix youtube channels upload
  77   const uploadsObject = info.find(i => !i.ie_key && !i.duration && i.title === 'Uploads')
  78
  79   if (uploadsObject) {
  80     console.log('Fixing URL to %s.', uploadsObject.url)
  81
  82     info = await getYoutubeDLInfo(youtubeDLBinary, uploadsObject.url, command.args)
  83   }
  84
  85   let infoArray: any[]
  86
  87   infoArray = [].concat(info)
  88   if (options.first) {
  89     infoArray = infoArray.slice(0, options.first)
  90   } else if (options.last) {
  91     infoArray = infoArray.slice(-options.last)
  92   }
  93
  94   log.info('Will download and upload %d videos.\n', infoArray.length)
  95
  96   let skipInterval = true
  97   for (const [ index, info ] of infoArray.entries()) {
  98     try {
  99       if (index > 0 && options.waitInterval && !skipInterval) {
 100         log.info('Wait for %d seconds before continuing.', options.waitInterval / 1000)
 101         await wait(options.waitInterval)
 102       }
 103
 104       skipInterval = await processVideo({
 105         cwd: options.tmpdir,
 106         url,
 107         username,
 108         password,
 109         youtubeInfo: info
 110       })
 111     } catch (err) {
 112       console.error('Cannot process video.', { info, url, err })
 113     }
 114   }
 115
 116   log.info('Video/s for user %s imported: %s', username, options.targetUrl)
 117   process.exit(0)
 118 }
 119
 120 async function processVideo (parameters: {
 121   cwd: string
 122   url: string
 123   username: string
 124   password: string
 125   youtubeInfo: any
 126 }) {
 127   const { youtubeInfo, cwd, url, username, password } = parameters
 128
 129   log.debug('Fetching object.', youtubeInfo)
 130
 131   const videoInfo = await fetchObject(youtubeInfo)
 132   log.debug('Fetched object.', videoInfo)
 133
 134   if (
 135     options.since &&
 136     videoInfo.originallyPublishedAtWithoutTime &&
 137     videoInfo.originallyPublishedAtWithoutTime.getTime() < options.since.getTime()
 138   ) {
 139     log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.since))
 140     return true
 141   }
 142
 143   if (
 144     options.until &&
 145     videoInfo.originallyPublishedAtWithoutTime &&
 146     videoInfo.originallyPublishedAtWithoutTime.getTime() > options.until.getTime()
 147   ) {
 148     log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.until))
 149     return true
 150   }
 151
 152   const server = buildServer(url)
 153   const { data } = await server.search.advancedVideoSearch({
 154     search: {
 155       search: videoInfo.name,
 156       sort: '-match',
 157       searchTarget: 'local'
 158     }
 159   })
 160
 161   log.info('############################################################\n')
 162
 163   if (data.find(v => v.name === videoInfo.name)) {
 164     log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.name)
 165     return true
 166   }
 167
 168   const path = join(cwd, sha256(videoInfo.url) + '.mp4')
 169
 170   log.info('Downloading video "%s"...', videoInfo.name)
 171
 172   try {
 173     const youtubeDLBinary = await YoutubeDLCLI.safeGet()
 174     const output = await youtubeDLBinary.download({
 175       url: videoInfo.url,
 176       format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
 177       output: path,
 178       additionalYoutubeDLArgs: command.args,
 179       processOptions
 180     })
 181
 182     log.info(output.join('\n'))
 183     await uploadVideoOnPeerTube({
 184       cwd,
 185       url,
 186       username,
 187       password,
 188       videoInfo,
 189       videoPath: path
 190     })
 191   } catch (err) {
 192     log.error(err.message)
 193   }
 194
 195   return false
 196 }
 197
 198 async function uploadVideoOnPeerTube (parameters: {
 199   videoInfo: YoutubeDLInfo
 200   videoPath: string
 201   cwd: string
 202   url: string
 203   username: string
 204   password: string
 205 }) {
 206   const { videoInfo, videoPath, cwd, url, username, password } = parameters
 207
 208   const server = buildServer(url)
 209   await assignToken(server, username, password)
 210
 211   let thumbnailfile: string
 212   if (videoInfo.thumbnailUrl) {
 213     thumbnailfile = join(cwd, sha256(videoInfo.thumbnailUrl) + '.jpg')
 214
 215     await doRequestAndSaveToFile(videoInfo.thumbnailUrl, thumbnailfile)
 216   }
 217
 218   const baseAttributes = await buildVideoAttributesFromCommander(server, program, videoInfo)
 219
 220   const attributes = {
 221     ...baseAttributes,
 222
 223     originallyPublishedAtWithoutTime: videoInfo.originallyPublishedAtWithoutTime
 224       ? videoInfo.originallyPublishedAtWithoutTime.toISOString()
 225       : null,
 226
 227     thumbnailfile,
 228     previewfile: thumbnailfile,
 229     fixture: videoPath
 230   }
 231
 232   log.info('\nUploading on PeerTube video "%s".', attributes.name)
 233
 234   try {
 235     await server.videos.upload({ attributes })
 236   } catch (err) {
 237     if (err.message.indexOf('401') !== -1) {
 238       log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.')
 239
 240       server.accessToken = await server.login.getAccessToken(username, password)
 241
 242       await server.videos.upload({ attributes })
 243     } else {
 244       exitError(err.message)
 245     }
 246   }
 247
 248   await remove(videoPath)
 249   if (thumbnailfile) await remove(thumbnailfile)
 250
 251   log.info('Uploaded video "%s"!\n', attributes.name)
 252 }
 253
 254 /* ---------------------------------------------------------- */
 255
 256 async function fetchObject (info: any) {
 257   const url = buildUrl(info)
 258
 259   const youtubeDLCLI = await YoutubeDLCLI.safeGet()
 260   const result = await youtubeDLCLI.getInfo({
 261     url,
 262     format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
 263     processOptions
 264   })
 265
 266   const builder = new YoutubeDLInfoBuilder(result)
 267
 268   const videoInfo = builder.getInfo()
 269
 270   return { ...videoInfo, url }
 271 }
 272
 273 function buildUrl (info: any) {
 274   const webpageUrl = info.webpage_url as string
 275   if (webpageUrl?.match(/^https?:\/\//)) return webpageUrl
 276
 277   const url = info.url as string
 278   if (url?.match(/^https?:\/\//)) return url
 279
 280   // It seems youtube-dl does not return the video url
 281   return 'https://www.youtube.com/watch?v=' + info.id
 282 }
 283
 284 function normalizeTargetUrl (url: string) {
 285   let normalizedUrl = url.replace(/\/+$/, '')
 286
 287   if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) {
 288     normalizedUrl = 'https://' + normalizedUrl
 289   }
 290
 291   return normalizedUrl
 292 }
 293
 294 async function promptPassword () {
 295   return new Promise<string>((res, rej) => {
 296     prompt.start()
 297     const schema = {
 298       properties: {
 299         password: {
 300           hidden: true,
 301           required: true
 302         }
 303       }
 304     }
 305     prompt.get(schema, function (err, result) {
 306       if (err) {
 307         return rej(err)
 308       }
 309       return res(result.password)
 310     })
 311   })
 312 }
 313
 314 function parseDate (dateAsStr: string): Date {
 315   if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) {
 316     exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`)
 317   }
 318   const date = new Date(dateAsStr)
 319   date.setHours(0, 0, 0)
 320   if (isNaN(date.getTime())) {
 321     exitError(`Invalid date passed: ${dateAsStr}. See help for usage.`)
 322   }
 323   return date
 324 }
 325
 326 function formatDate (date: Date): string {
 327   return date.toISOString().split('T')[0]
 328 }
 329
 330 function convertIntoMs (secondsAsStr: string): number {
 331   const seconds = parseInt(secondsAsStr, 10)
 332   if (seconds <= 0) {
 333     exitError(`Invalid duration passed: ${seconds}. Expected duration to be strictly positive and in seconds`)
 334   }
 335   return Math.round(seconds * 1000)
 336 }
 337
 338 function exitError (message: string, ...meta: any[]) {
 339   // use console.error instead of log.error here
 340   console.error(message, ...meta)
 341   process.exit(-1)
 342 }
 343
 344 function getYoutubeDLInfo (youtubeDLCLI: YoutubeDLCLI, url: string, args: string[]) {
 345   return youtubeDLCLI.getInfo({
 346     url,
 347     format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
 348     additionalYoutubeDLArgs: [ '-j', '--flat-playlist', '--playlist-reverse', ...args ],
 349     processOptions
 350   })
 351 }