server/tools/peertube-import-videos.ts

   1 import { program } from 'commander'
   2 import { accessSync, constants } from 'fs'
   3 import { remove } from 'fs-extra'
   4 import { join } from 'path'
   5 import { YoutubeDLCLI, YoutubeDLInfo, YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl'
   6 import { wait } from '@shared/core-utils'
   7 import { sha256 } from '@shared/extra-utils'
   8 import { doRequestAndSaveToFile } from '../helpers/requests'
   9 import {
  10   assignToken,
  11   buildCommonVideoOptions,
  12   buildServer,
  13   buildVideoAttributesFromCommander,
  14   getLogger,
  15   getServerCredentials
  16 } from './cli'
  17
  18 import prompt = require('prompt')
  19
  20 const processOptions = {
  21   maxBuffer: Infinity
  22 }
  23
  24 let command = program
  25   .name('import-videos')
  26
  27 command = buildCommonVideoOptions(command)
  28
  29 command
  30   .option('-u, --url <url>', 'Server url')
  31   .option('-U, --username <username>', 'Username')
  32   .option('-p, --password <token>', 'Password')
  33   .option('--target-url <targetUrl>', 'Video target URL')
  34   .option('--since <since>', 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)', parseDate)
  35   .option('--until <until>', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate)
  36   .option('--first <first>', 'Process first n elements of returned playlist')
  37   .option('--last <last>', 'Process last n elements of returned playlist')
  38   .option('--wait-interval <waitInterval>', 'Duration between two video imports (in seconds)', convertIntoMs)
  39   .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname)
  40   .usage('[global options] [ -- youtube-dl options]')
  41   .parse(process.argv)
  42
  43 const options = command.opts()
  44
  45 const log = getLogger(options.verbose)
  46
  47 getServerCredentials(command)
  48   .then(({ url, username, password }) => {
  49     if (!options.targetUrl) {
  50       exitError('--target-url field is required.')
  51     }
  52
  53     try {
  54       accessSync(options.tmpdir, constants.R_OK | constants.W_OK)
  55     } catch (e) {
  56       exitError('--tmpdir %s: directory does not exist or is not accessible', options.tmpdir)
  57     }
  58
  59     url = normalizeTargetUrl(url)
  60     options.targetUrl = normalizeTargetUrl(options.targetUrl)
  61
  62     run(url, username, password)
  63       .catch(err => exitError(err))
  64   })
  65   .catch(err => console.error(err))
  66
  67 async function run (url: string, username: string, password: string) {
  68   if (!password) password = await promptPassword()
  69
  70   const youtubeDLBinary = await YoutubeDLCLI.safeGet()
  71
  72   let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args)
  73
  74   if (!Array.isArray(info)) info = [ info ]
  75
  76   // Try to fix youtube channels upload
  77   const uploadsObject = info.find(i => !i.ie_key && !i.duration && i.title === 'Uploads')
  78
  79   if (uploadsObject) {
  80     console.log('Fixing URL to %s.', uploadsObject.url)
  81
  82     info = await getYoutubeDLInfo(youtubeDLBinary, uploadsObject.url, command.args)
  83   }
  84
  85   let infoArray: any[]
  86
  87   infoArray = [].concat(info)
  88   if (options.first) {
  89     infoArray = infoArray.slice(0, options.first)
  90   } else if (options.last) {
  91     infoArray = infoArray.slice(-options.last)
  92   }
  93
  94   log.info('Will download and upload %d videos.\n', infoArray.length)
  95
  96   let skipInterval = true
  97   for (const [ index, info ] of infoArray.entries()) {
  98     try {
  99       if (index > 0 && options.waitInterval && !skipInterval) {
 100         log.info('Wait for %d seconds before continuing.', options.waitInterval / 1000)
 101         await wait(options.waitInterval)
 102       }
 103
 104       skipInterval = await processVideo({
 105         cwd: options.tmpdir,
 106         url,
 107         username,
 108         password,
 109         youtubeInfo: info
 110       })
 111     } catch (err) {
 112       console.error('Cannot process video.', { info, url, err })
 113     }
 114   }
 115
 116   log.info('Video/s for user %s imported: %s', username, options.targetUrl)
 117   process.exit(0)
 118 }
 119
 120 async function processVideo (parameters: {
 121   cwd: string
 122   url: string
 123   username: string
 124   password: string
 125   youtubeInfo: any
 126 }) {
 127   const { youtubeInfo, cwd, url, username, password } = parameters
 128
 129   log.debug('Fetching object.', youtubeInfo)
 130
 131   const videoInfo = await fetchObject(youtubeInfo)
 132   log.debug('Fetched object.', videoInfo)
 133
 134   if (options.since && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() < options.since.getTime()) {
 135     log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.since))
 136     return true
 137   }
 138
 139   if (options.until && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() > options.until.getTime()) {
 140     log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.until))
 141     return true
 142   }
 143
 144   const server = buildServer(url)
 145   const { data } = await server.search.advancedVideoSearch({
 146     search: {
 147       search: videoInfo.name,
 148       sort: '-match',
 149       searchTarget: 'local'
 150     }
 151   })
 152
 153   log.info('############################################################\n')
 154
 155   if (data.find(v => v.name === videoInfo.name)) {
 156     log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.name)
 157     return true
 158   }
 159
 160   const path = join(cwd, sha256(videoInfo.url) + '.mp4')
 161
 162   log.info('Downloading video "%s"...', videoInfo.name)
 163
 164   try {
 165     const youtubeDLBinary = await YoutubeDLCLI.safeGet()
 166     const output = await youtubeDLBinary.download({
 167       url: videoInfo.url,
 168       format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
 169       output: path,
 170       additionalYoutubeDLArgs: command.args,
 171       processOptions
 172     })
 173
 174     log.info(output.join('\n'))
 175     await uploadVideoOnPeerTube({
 176       cwd,
 177       url,
 178       username,
 179       password,
 180       videoInfo,
 181       videoPath: path
 182     })
 183   } catch (err) {
 184     log.error(err.message)
 185   }
 186
 187   return false
 188 }
 189
 190 async function uploadVideoOnPeerTube (parameters: {
 191   videoInfo: YoutubeDLInfo
 192   videoPath: string
 193   cwd: string
 194   url: string
 195   username: string
 196   password: string
 197 }) {
 198   const { videoInfo, videoPath, cwd, url, username, password } = parameters
 199
 200   const server = buildServer(url)
 201   await assignToken(server, username, password)
 202
 203   let thumbnailfile: string
 204   if (videoInfo.thumbnailUrl) {
 205     thumbnailfile = join(cwd, sha256(videoInfo.thumbnailUrl) + '.jpg')
 206
 207     await doRequestAndSaveToFile(videoInfo.thumbnailUrl, thumbnailfile)
 208   }
 209
 210   const baseAttributes = await buildVideoAttributesFromCommander(server, program, videoInfo)
 211
 212   const attributes = {
 213     ...baseAttributes,
 214
 215     originallyPublishedAt: videoInfo.originallyPublishedAt
 216       ? videoInfo.originallyPublishedAt.toISOString()
 217       : null,
 218
 219     thumbnailfile,
 220     previewfile: thumbnailfile,
 221     fixture: videoPath
 222   }
 223
 224   log.info('\nUploading on PeerTube video "%s".', attributes.name)
 225
 226   try {
 227     await server.videos.upload({ attributes })
 228   } catch (err) {
 229     if (err.message.indexOf('401') !== -1) {
 230       log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.')
 231
 232       server.accessToken = await server.login.getAccessToken(username, password)
 233
 234       await server.videos.upload({ attributes })
 235     } else {
 236       exitError(err.message)
 237     }
 238   }
 239
 240   await remove(videoPath)
 241   if (thumbnailfile) await remove(thumbnailfile)
 242
 243   log.info('Uploaded video "%s"!\n', attributes.name)
 244 }
 245
 246 /* ---------------------------------------------------------- */
 247
 248 async function fetchObject (info: any) {
 249   const url = buildUrl(info)
 250
 251   const youtubeDLCLI = await YoutubeDLCLI.safeGet()
 252   const result = await youtubeDLCLI.getInfo({
 253     url,
 254     format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
 255     processOptions
 256   })
 257
 258   const builder = new YoutubeDLInfoBuilder(result)
 259
 260   const videoInfo = builder.getInfo()
 261
 262   return { ...videoInfo, url }
 263 }
 264
 265 function buildUrl (info: any) {
 266   const webpageUrl = info.webpage_url as string
 267   if (webpageUrl?.match(/^https?:\/\//)) return webpageUrl
 268
 269   const url = info.url as string
 270   if (url?.match(/^https?:\/\//)) return url
 271
 272   // It seems youtube-dl does not return the video url
 273   return 'https://www.youtube.com/watch?v=' + info.id
 274 }
 275
 276 function normalizeTargetUrl (url: string) {
 277   let normalizedUrl = url.replace(/\/+$/, '')
 278
 279   if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) {
 280     normalizedUrl = 'https://' + normalizedUrl
 281   }
 282
 283   return normalizedUrl
 284 }
 285
 286 async function promptPassword () {
 287   return new Promise<string>((res, rej) => {
 288     prompt.start()
 289     const schema = {
 290       properties: {
 291         password: {
 292           hidden: true,
 293           required: true
 294         }
 295       }
 296     }
 297     prompt.get(schema, function (err, result) {
 298       if (err) {
 299         return rej(err)
 300       }
 301       return res(result.password)
 302     })
 303   })
 304 }
 305
 306 function parseDate (dateAsStr: string): Date {
 307   if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) {
 308     exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`)
 309   }
 310   const date = new Date(dateAsStr)
 311   date.setHours(0, 0, 0)
 312   if (isNaN(date.getTime())) {
 313     exitError(`Invalid date passed: ${dateAsStr}. See help for usage.`)
 314   }
 315   return date
 316 }
 317
 318 function formatDate (date: Date): string {
 319   return date.toISOString().split('T')[0]
 320 }
 321
 322 function convertIntoMs (secondsAsStr: string): number {
 323   const seconds = parseInt(secondsAsStr, 10)
 324   if (seconds <= 0) {
 325     exitError(`Invalid duration passed: ${seconds}. Expected duration to be strictly positive and in seconds`)
 326   }
 327   return Math.round(seconds * 1000)
 328 }
 329
 330 function exitError (message: string, ...meta: any[]) {
 331   // use console.error instead of log.error here
 332   console.error(message, ...meta)
 333   process.exit(-1)
 334 }
 335
 336 function getYoutubeDLInfo (youtubeDLCLI: YoutubeDLCLI, url: string, args: string[]) {
 337   return youtubeDLCLI.getInfo({
 338     url,
 339     format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
 340     additionalYoutubeDLArgs: [ '-j', '--flat-playlist', '--playlist-reverse', ...args ],
 341     processOptions
 342   })
 343 }