1 import { registerTSPaths } from '../helpers/register-ts-paths'
4 import { program } from 'commander'
5 import { accessSync, constants } from 'fs'
6 import { remove } from 'fs-extra'
7 import { join } from 'path'
8 import { sha256 } from '../helpers/core-utils'
9 import { doRequestAndSaveToFile } from '../helpers/requests'
12 buildCommonVideoOptions,
14 buildVideoAttributesFromCommander,
18 import { wait } from '@shared/extra-utils'
19 import { YoutubeDLCLI, YoutubeDLInfo, YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl'
20 import prompt = require('prompt')
22 const processOptions = {
27 .name('import-videos')
29 command = buildCommonVideoOptions(command)
32 .option('-u, --url <url>', 'Server url')
33 .option('-U, --username <username>', 'Username')
34 .option('-p, --password <token>', 'Password')
35 .option('--target-url <targetUrl>', 'Video target URL')
36 .option('--since <since>', 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)', parseDate)
37 .option('--until <until>', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate)
38 .option('--first <first>', 'Process first n elements of returned playlist')
39 .option('--last <last>', 'Process last n elements of returned playlist')
40 .option('--wait-interval <waitInterval>', 'Duration between two video imports (in seconds)', convertIntoMs)
41 .option('-T, --tmpdir <tmpdir>', 'Working directory', __dirname)
42 .usage("[global options] [ -- youtube-dl options]")
45 const options = command.opts()
47 const log = getLogger(options.verbose)
49 getServerCredentials(command)
50 .then(({ url, username, password }) => {
51 if (!options.targetUrl) {
52 exitError('--target-url field is required.')
56 accessSync(options.tmpdir, constants.R_OK | constants.W_OK)
58 exitError('--tmpdir %s: directory does not exist or is not accessible', options.tmpdir)
61 url = normalizeTargetUrl(url)
62 options.targetUrl = normalizeTargetUrl(options.targetUrl)
64 run(url, username, password)
65 .catch(err => exitError(err))
67 .catch(err => console.error(err))
69 async function run (url: string, username: string, password: string) {
70 if (!password) password = await promptPassword()
72 const youtubeDLBinary = await YoutubeDLCLI.safeGet()
74 let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args)
76 if (!Array.isArray(info)) info = [ info ]
78 // Try to fix youtube channels upload
79 const uploadsObject = info.find(i => !i.ie_key && !i.duration && i.title === 'Uploads')
82 console.log('Fixing URL to %s.', uploadsObject.url)
84 info = await getYoutubeDLInfo(youtubeDLBinary, uploadsObject.url, command.args)
89 infoArray = [].concat(info)
91 infoArray = infoArray.slice(0, options.first)
92 } else if (options.last) {
93 infoArray = infoArray.slice(-options.last)
96 log.info('Will download and upload %d videos.\n', infoArray.length)
98 for (const [ index, info ] of infoArray.entries()) {
100 if (index > 0 && options.waitInterval) {
101 log.info("Wait for %d seconds before continuing.", options.waitInterval / 1000)
102 await wait(options.waitInterval)
113 console.error('Cannot process video.', { info, url, err })
117 log.info('Video/s for user %s imported: %s', username, options.targetUrl)
121 async function processVideo (parameters: {
128 const { youtubeInfo, cwd, url, username, password } = parameters
130 log.debug('Fetching object.', youtubeInfo)
132 const videoInfo = await fetchObject(youtubeInfo)
133 log.debug('Fetched object.', videoInfo)
135 if (options.since && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() < options.since.getTime()) {
136 log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.since))
140 if (options.until && videoInfo.originallyPublishedAt && videoInfo.originallyPublishedAt.getTime() > options.until.getTime()) {
141 log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.until))
145 const server = buildServer(url)
146 const { data } = await server.search.advancedVideoSearch({
148 search: videoInfo.name,
150 searchTarget: 'local'
154 log.info('############################################################\n')
156 if (data.find(v => v.name === videoInfo.name)) {
157 log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.name)
161 const path = join(cwd, sha256(videoInfo.url) + '.mp4')
163 log.info('Downloading video "%s"...', videoInfo.name)
166 const youtubeDLBinary = await YoutubeDLCLI.safeGet()
167 const output = await youtubeDLBinary.download({
169 format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
171 additionalYoutubeDLArgs: command.args,
175 log.info(output.join('\n'))
176 await uploadVideoOnPeerTube({
185 log.error(err.message)
189 async function uploadVideoOnPeerTube (parameters: {
190 videoInfo: YoutubeDLInfo
197 const { videoInfo, videoPath, cwd, url, username, password } = parameters
199 const server = buildServer(url)
200 await assignToken(server, username, password)
202 let thumbnailfile: string
203 if (videoInfo.thumbnailUrl) {
204 thumbnailfile = join(cwd, sha256(videoInfo.thumbnailUrl) + '.jpg')
206 await doRequestAndSaveToFile(videoInfo.thumbnailUrl, thumbnailfile)
209 const baseAttributes = await buildVideoAttributesFromCommander(server, program, videoInfo)
214 originallyPublishedAt: videoInfo.originallyPublishedAt
215 ? videoInfo.originallyPublishedAt.toISOString()
219 previewfile: thumbnailfile,
223 log.info('\nUploading on PeerTube video "%s".', attributes.name)
226 await server.videos.upload({ attributes })
228 if (err.message.indexOf('401') !== -1) {
229 log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.')
231 server.accessToken = await server.login.getAccessToken(username, password)
233 await server.videos.upload({ attributes })
235 exitError(err.message)
239 await remove(videoPath)
240 if (thumbnailfile) await remove(thumbnailfile)
242 log.info('Uploaded video "%s"!\n', attributes.name)
245 /* ---------------------------------------------------------- */
247 async function fetchObject (info: any) {
248 const url = buildUrl(info)
250 const youtubeDLCLI = await YoutubeDLCLI.safeGet()
251 const result = await youtubeDLCLI.getInfo({
253 format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
257 const builder = new YoutubeDLInfoBuilder(result)
259 const videoInfo = builder.getInfo()
261 return { ...videoInfo, url }
264 function buildUrl (info: any) {
265 const webpageUrl = info.webpage_url as string
266 if (webpageUrl?.match(/^https?:\/\//)) return webpageUrl
268 const url = info.url as string
269 if (url?.match(/^https?:\/\//)) return url
271 // It seems youtube-dl does not return the video url
272 return 'https://www.youtube.com/watch?v=' + info.id
275 function normalizeTargetUrl (url: string) {
276 let normalizedUrl = url.replace(/\/+$/, '')
278 if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) {
279 normalizedUrl = 'https://' + normalizedUrl
285 async function promptPassword () {
286 return new Promise<string>((res, rej) => {
296 prompt.get(schema, function (err, result) {
300 return res(result.password)
305 function parseDate (dateAsStr: string): Date {
306 if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) {
307 exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`)
309 const date = new Date(dateAsStr)
310 date.setHours(0, 0, 0)
311 if (isNaN(date.getTime())) {
312 exitError(`Invalid date passed: ${dateAsStr}. See help for usage.`)
317 function formatDate (date: Date): string {
318 return date.toISOString().split('T')[0]
321 function convertIntoMs (secondsAsStr: string): number {
322 const seconds = parseInt(secondsAsStr, 10)
324 exitError(`Invalid duration passed: ${seconds}. Expected duration to be strictly positive and in seconds`)
326 return Math.round(seconds * 1000)
329 function exitError (message: string, ...meta: any[]) {
330 // use console.error instead of log.error here
331 console.error(message, ...meta)
335 function getYoutubeDLInfo (youtubeDLCLI: YoutubeDLCLI, url: string, args: string[]) {
336 return youtubeDLCLI.getInfo({
338 format: YoutubeDLCLI.getYoutubeDLVideoFormat([]),
339 additionalYoutubeDLArgs: [ '-j', '--flat-playlist', '--playlist-reverse', ...args ],