server/tools/peertube-import-videos.ts

   1 import { registerTSPaths } from '../helpers/register-ts-paths'
   2 registerTSPaths()
   3
   4 import { program } from 'commander'
   5 import { accessSync, constants } from 'fs'
   6 import { remove } from 'fs-extra'
   7 import { join } from 'path'
   8 import { sha256 } from '@shared/extra-utils'
   9 import { wait } from '@shared/core-utils'
  10 import { doRequestAndSaveToFile } from '../helpers/requests'
  11 import {
  12   assignToken,
  13   buildCommonVideoOptions,
  14   buildServer,
  15   buildVideoAttributesFromCommander,
  16   getLogger,
  17   getServerCredentials
  18 } from './cli'
  19 import { YoutubeDLCLI, YoutubeDLInfo, YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl'
  20 import prompt = require('prompt')
  21
  22 const processOptions = {
  23   maxBuffer: Infinity
  24 }
  25
  26 let command = program
  27   .name('import-videos')
  28
  29 command = buildCommonVideoOptions(command)
  30
  31 command
  32   .option('-u, --url <url>', 'Server url')
  33   .option('-U, --username <username>', 'Username')
  34   .option('-p, --password <token>', 'Password')
  35   .option('--target-url <targetUrl>', 'Video target URL')
  36   .option('--since <since>', 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)', parseDate)
  37   .option('--until <until>', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate)
  38   .option('--first <first>', 'Process first n elements of returned playlist')
  39   .option('--last <last>', 'Process last n elements of returned playlist')
  40   .option('--wait-interval <waitInterval>', 'Duration between two video imports (in seconds)', convertIntoMs)
  41   .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname)
  42   .usage("[global options] [ -- youtube-dl options]")
  43   .parse(process.argv)
  44
  45 const options = command.opts()
  46
  47 const log = getLogger(options.verbose)
  48
  49 getServerCredentials(command)
  50   .then(({ url, username, password }) => {
  51     if (!options.targetUrl) {
  52       exitError('--target-url field is required.')
  53     }
  54
  55     try {
  56       accessSync(options.tmpdir, constants.R_OK | constants.W_OK)
  57     } catch (e) {
  58       exitError('--tmpdir %s: directory does not exist or is not accessible', options.tmpdir)
  59     }
  60
  61     url = normalizeTargetUrl(url)
  62     options.targetUrl = normalizeTargetUrl(options.targetUrl)
  63
  64     run(url, username, password)
  65       .catch(err => exitError(err))
  66   })
  67   .catch(err => console.error(err))
  68
  69 async function run (url: string, username: string, password: string) {
  70   if (!password) password = await promptPassword()
  71
  72   const youtubeDLBinary = await YoutubeDLCLI.safeGet()
  73
  74   let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args)
  75
  76   if (!Array.isArray(info)) info = [ info ]
  77
  78   // Try to fix youtube channels upload
  79   const uploadsObject = info.find(i => !i.ie_key && !i.duration && i.title === 'Uploads')
  80
  81   if (uploadsObject) {
  82     console.log('Fixing URL to %s.', uploadsObject.url)
  83
  84     info = await getYoutubeDLInfo(youtubeDLBinary, uploadsObject.url, command.args)
  85   }
  86
  87   let infoArray: any[]
  88
  89   infoArray = [].concat(info)
  90   if (options.first) {
  91     infoArray = infoArray.slice(0, options.first)
  92   } else if (options.last) {
  93     infoArray = infoArray.slice(-options.last)
  94   }
  95
  96   log.info('Will download and upload %d videos.\n', infoArray.length)
  97
  98   let skipInterval = true
  99   for (const [ index, info ] of infoArray.entries()) {
 100     try {
 101       if (index > 0 && options.waitInterval && !skipInterval) {
 102         log.info("Wait for %d seconds before continuing.", options.waitInterval / 1000)
 103         await wait(options.waitInterval)
 104       }
 105
 106       skipInterval = await processVideo({
 107         cwd: options.tmpdir,
 108         url,
 109         username,
 110         password,
 111         youtubeInfo: info
 112       })
 113     } catch (err) {
 114       console.error('Cannot process video.', { info, url, err })
 115     }
 116   }
 117
 118   log.info('Video/s for user %s imported: %s', username, options.targetUrl)
 119   process.exit(0)
 120 }
 121
 122 async function processVideo (parameters: {
 123   cwd: string
 124   url: string
 125   username: string
 126   password: string
 127   youtubeInfo: any
 128 }) {
 129   const { youtubeInfo, cwd, url, username, password } = parameters
 130
 131   log.debug('Fetching object.', youtubeInfo)
 132
 133   const videoInfo = await fetchObject(youtubeInfo)
 134   log.debug('Fetched object.', videoInfo)
 135
 136   if (options.since && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() < options.since.getTime()) {
 137     log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.since))
 138     return true
 139   }
 140
 141   if (options.until && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() > options.until.getTime()) {
 142     log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.until))
 143     return true
 144   }
 145
 146   const server = buildServer(url)
 147   const { data } = await server.search.advancedVideoSearch({
 148     search: {
 149       search: videoInfo.name,
 150       sort: '-match',
 151       searchTarget: 'local'
 152     }
 153   })
 154
 155   log.info('############################################################\n')
 156
 157   if (data.find(v => v.name === videoInfo.name)) {
 158     log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.name)
 159     return true
 160   }
 161
 162   const path = join(cwd, sha256(videoInfo.url) + '.mp4')
 163
 164   log.info('Downloading video "%s"...', videoInfo.name)
 165
 166   try {
 167     const youtubeDLBinary = await YoutubeDLCLI.safeGet()
 168     const output = await youtubeDLBinary.download({
 169       url: videoInfo.url,
 170       format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
 171       output: path,
 172       additionalYoutubeDLArgs: command.args,
 173       processOptions
 174     })
 175
 176     log.info(output.join('\n'))
 177     await uploadVideoOnPeerTube({
 178       cwd,
 179       url,
 180       username,
 181       password,
 182       videoInfo,
 183       videoPath: path
 184     })
 185   } catch (err) {
 186     log.error(err.message)
 187   }
 188
 189   return false
 190 }
 191
 192 async function uploadVideoOnPeerTube (parameters: {
 193   videoInfo: YoutubeDLInfo
 194   videoPath: string
 195   cwd: string
 196   url: string
 197   username: string
 198   password: string
 199 }) {
 200   const { videoInfo, videoPath, cwd, url, username, password } = parameters
 201
 202   const server = buildServer(url)
 203   await assignToken(server, username, password)
 204
 205   let thumbnailfile: string
 206   if (videoInfo.thumbnailUrl) {
 207     thumbnailfile = join(cwd, sha256(videoInfo.thumbnailUrl) + '.jpg')
 208
 209     await doRequestAndSaveToFile(videoInfo.thumbnailUrl, thumbnailfile)
 210   }
 211
 212   const baseAttributes = await buildVideoAttributesFromCommander(server, program, videoInfo)
 213
 214   const attributes = {
 215     ...baseAttributes,
 216
 217     originallyPublishedAt: videoInfo.originallyPublishedAt
 218       ? videoInfo.originallyPublishedAt.toISOString()
 219       : null,
 220
 221     thumbnailfile,
 222     previewfile: thumbnailfile,
 223     fixture: videoPath
 224   }
 225
 226   log.info('\nUploading on PeerTube video "%s".', attributes.name)
 227
 228   try {
 229     await server.videos.upload({ attributes })
 230   } catch (err) {
 231     if (err.message.indexOf('401') !== -1) {
 232       log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.')
 233
 234       server.accessToken = await server.login.getAccessToken(username, password)
 235
 236       await server.videos.upload({ attributes })
 237     } else {
 238       exitError(err.message)
 239     }
 240   }
 241
 242   await remove(videoPath)
 243   if (thumbnailfile) await remove(thumbnailfile)
 244
 245   log.info('Uploaded video "%s"!\n', attributes.name)
 246 }
 247
 248 /* ---------------------------------------------------------- */
 249
 250 async function fetchObject (info: any) {
 251   const url = buildUrl(info)
 252
 253   const youtubeDLCLI = await YoutubeDLCLI.safeGet()
 254   const result = await youtubeDLCLI.getInfo({
 255     url,
 256     format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
 257     processOptions
 258   })
 259
 260   const builder = new YoutubeDLInfoBuilder(result)
 261
 262   const videoInfo = builder.getInfo()
 263
 264   return { ...videoInfo, url }
 265 }
 266
 267 function buildUrl (info: any) {
 268   const webpageUrl = info.webpage_url as string
 269   if (webpageUrl?.match(/^https?:\/\//)) return webpageUrl
 270
 271   const url = info.url as string
 272   if (url?.match(/^https?:\/\//)) return url
 273
 274   // It seems youtube-dl does not return the video url
 275   return 'https://www.youtube.com/watch?v=' + info.id
 276 }
 277
 278 function normalizeTargetUrl (url: string) {
 279   let normalizedUrl = url.replace(/\/+$/, '')
 280
 281   if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) {
 282     normalizedUrl = 'https://' + normalizedUrl
 283   }
 284
 285   return normalizedUrl
 286 }
 287
 288 async function promptPassword () {
 289   return new Promise<string>((res, rej) => {
 290     prompt.start()
 291     const schema = {
 292       properties: {
 293         password: {
 294           hidden: true,
 295           required: true
 296         }
 297       }
 298     }
 299     prompt.get(schema, function (err, result) {
 300       if (err) {
 301         return rej(err)
 302       }
 303       return res(result.password)
 304     })
 305   })
 306 }
 307
 308 function parseDate (dateAsStr: string): Date {
 309   if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) {
 310     exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`)
 311   }
 312   const date = new Date(dateAsStr)
 313   date.setHours(0, 0, 0)
 314   if (isNaN(date.getTime())) {
 315     exitError(`Invalid date passed: ${dateAsStr}. See help for usage.`)
 316   }
 317   return date
 318 }
 319
 320 function formatDate (date: Date): string {
 321   return date.toISOString().split('T')[0]
 322 }
 323
 324 function convertIntoMs (secondsAsStr: string): number {
 325   const seconds = parseInt(secondsAsStr, 10)
 326   if (seconds <= 0) {
 327     exitError(`Invalid duration passed: ${seconds}. Expected duration to be strictly positive and in seconds`)
 328   }
 329   return Math.round(seconds * 1000)
 330 }
 331
 332 function exitError (message: string, ...meta: any[]) {
 333   // use console.error instead of log.error here
 334   console.error(message, ...meta)
 335   process.exit(-1)
 336 }
 337
 338 function getYoutubeDLInfo (youtubeDLCLI: YoutubeDLCLI, url: string, args: string[]) {
 339   return youtubeDLCLI.getInfo({
 340     url,
 341     format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
 342     additionalYoutubeDLArgs: [ '-j', '--flat-playlist', '--playlist-reverse', ...args ],
 343     processOptions
 344   })
 345 }