X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Ftools%2Fpeertube-import-videos.ts;h=b3c3aee3e496a975bf8a4ad6c2e26df7022ebf73;hb=49c3bf6fa25afb49c8a27937147043c6e4ce95c3;hp=9a366dbbda3bca26376bb11f3f577a8a5eeafe87;hpb=97567dd81f508dd6295ac4d73d849aa2ce0a6549;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts index 9a366dbbd..b3c3aee3e 100644 --- a/server/tools/peertube-import-videos.ts +++ b/server/tools/peertube-import-videos.ts @@ -1,169 +1,190 @@ +import { registerTSPaths } from '../helpers/register-ts-paths' +registerTSPaths() + // FIXME: https://github.com/nodejs/node/pull/16853 require('tls').DEFAULT_ECDH_CURVE = 'auto' import * as program from 'commander' import { join } from 'path' -import { VideoPrivacy } from '../../shared/models/videos' import { doRequestAndSaveToFile } from '../helpers/requests' import { CONSTRAINTS_FIELDS } from '../initializers/constants' import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../../shared/extra-utils/index' import { truncate } from 'lodash' import * as prompt from 'prompt' +import { accessSync, constants } from 'fs' import { remove } from 'fs-extra' import { sha256 } from '../helpers/core-utils' import { buildOriginallyPublishedAt, safeGetYoutubeDL } from '../helpers/youtube-dl' -import { getNetrc, getRemoteObjectOrDie, getSettings } from './cli' +import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getServerCredentials, getLogger } from './cli' -let accessToken: string -let client: { id: string, secret: string } +type UserInfo = { + username: string + password: string +} const processOptions = { - cwd: __dirname, maxBuffer: Infinity } -program +let command = program .name('import-videos') + +command = buildCommonVideoOptions(command) + +command .option('-u, --url ', 'Server url') .option('-U, --username ', 'Username') .option('-p, --password ', 'Password') - .option('-t, --target-url ', 'Video target URL') - .option('-l, --language ', 'Language ISO 639 code (fr or en...)') - .option('-v, --verbose', 'Verbose mode') + .option('--target-url ', 'Video target URL') + .option('--since ', 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)', parseDate) + .option('--until ', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate) + .option('--first ', 'Process first n elements of returned playlist') + .option('--last ', 'Process last n elements of returned playlist') + .option('-T, --tmpdir ', 'Working directory', __dirname) .parse(process.argv) -Promise.all([ getSettings(), getNetrc() ]) - .then(([ settings, netrc ]) => { - const { url, username, password } = getRemoteObjectOrDie(program, settings) - - if (!program[ 'targetUrl' ]) { - console.error('--targetUrl field is required.') +let log = getLogger(program[ 'verbose' ]) - process.exit(-1) - } +getServerCredentials(command) + .then(({ url, username, password }) => { + if (!program[ 'targetUrl' ]) { + exitError('--target-url field is required.') + } - removeEndSlashes(url) - removeEndSlashes(program[ 'targetUrl' ]) + try { + accessSync(program[ 'tmpdir' ], constants.R_OK | constants.W_OK) + } catch (e) { + exitError('--tmpdir %s: directory does not exist or is not accessible', program[ 'tmpdir' ]) + } - const user = { - username: username, - password: password - } + url = removeEndSlashes(url) + program[ 'targetUrl' ] = removeEndSlashes(program[ 'targetUrl' ]) - run(user, url) - .catch(err => { - console.error(err) - process.exit(-1) - }) - }) + const user = { username, password } -async function promptPassword () { - return new Promise((res, rej) => { - prompt.start() - const schema = { - properties: { - password: { - hidden: true, - required: true - } - } - } - prompt.get(schema, function (err, result) { - if (err) { - return rej(err) - } - return res(result.password) - }) + run(url, user) + .catch(err => { + exitError(err) + }) }) -} -async function run (user, url: string) { +async function run (url: string, user: UserInfo) { if (!user.password) { user.password = await promptPassword() } - const res = await getClient(url) - client = { - id: res.body.client_id, - secret: res.body.client_secret - } - - try { - const res = await login(program[ 'url' ], client, user) - accessToken = res.body.access_token - } catch (err) { - throw new Error('Cannot authenticate. Please check your username/password.') - } - const youtubeDL = await safeGetYoutubeDL() const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] youtubeDL.getInfo(program[ 'targetUrl' ], options, processOptions, async (err, info) => { if (err) { - console.log(err.message) - process.exit(1) + exitError(err.message) } let infoArray: any[] // Normalize utf8 fields - if (Array.isArray(info) === true) { - infoArray = info.map(i => normalizeObject(i)) - } else { - infoArray = [ normalizeObject(info) ] + infoArray = [].concat(info); + if (program[ 'first' ]) { + infoArray = infoArray.slice(0, program[ 'first' ]) + } else if (program[ 'last' ]) { + infoArray = infoArray.slice(- program[ 'last' ]) } - console.log('Will download and upload %d videos.\n', infoArray.length) + infoArray = infoArray.map(i => normalizeObject(i)) + + log.info('Will download and upload %d videos.\n', infoArray.length) for (const info of infoArray) { - await processVideo(info, program[ 'language' ], processOptions.cwd, url, user) + await processVideo({ + cwd: program[ 'tmpdir' ], + url, + user, + youtubeInfo: info + }) } - console.log('Video/s for user %s imported: %s', program[ 'username' ], program[ 'targetUrl' ]) + log.info('Video/s for user %s imported: %s', user.username, program[ 'targetUrl' ]) process.exit(0) }) } -function processVideo (info: any, languageCode: string, cwd: string, url: string, user) { +function processVideo (parameters: { + cwd: string, + url: string, + user: { username: string, password: string }, + youtubeInfo: any +}) { + const { youtubeInfo, cwd, url, user } = parameters + return new Promise(async res => { - if (program[ 'verbose' ]) console.log('Fetching object.', info) + log.debug('Fetching object.', youtubeInfo) - const videoInfo = await fetchObject(info) - if (program[ 'verbose' ]) console.log('Fetched object.', videoInfo) + const videoInfo = await fetchObject(youtubeInfo) + log.debug('Fetched object.', videoInfo) + + if (program[ 'since' ]) { + if (buildOriginallyPublishedAt(videoInfo).getTime() < program[ 'since' ].getTime()) { + log.info('Video "%s" has been published before "%s", don\'t upload it.\n', + videoInfo.title, formatDate(program[ 'since' ])); + return res(); + } + } + if (program[ 'until' ]) { + if (buildOriginallyPublishedAt(videoInfo).getTime() > program[ 'until' ].getTime()) { + log.info('Video "%s" has been published after "%s", don\'t upload it.\n', + videoInfo.title, formatDate(program[ 'until' ])); + return res(); + } + } const result = await searchVideoWithSort(url, videoInfo.title, '-match') - console.log('############################################################\n') + log.info('############################################################\n') if (result.body.data.find(v => v.name === videoInfo.title)) { - console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title) + log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title) return res() } const path = join(cwd, sha256(videoInfo.url) + '.mp4') - console.log('Downloading video "%s"...', videoInfo.title) + log.info('Downloading video "%s"...', videoInfo.title) const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] try { const youtubeDL = await safeGetYoutubeDL() youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => { if (err) { - console.error(err) + log.error(err) return res() } - console.log(output.join('\n')) - await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, cwd, url, user, languageCode) + log.info(output.join('\n')) + await uploadVideoOnPeerTube({ + cwd, + url, + user, + videoInfo: normalizeObject(videoInfo), + videoPath: path + }) return res() }) } catch (err) { - console.log(err.message) + log.error(err.message) return res() } }) } -async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: string, url: string, user, language?: string) { +async function uploadVideoOnPeerTube (parameters: { + videoInfo: any, + videoPath: string, + cwd: string, + url: string, + user: { username: string; password: string } +}) { + const { videoInfo, videoPath, cwd, url, user } = parameters + const category = await getCategory(videoInfo.categories, url) const licence = getLicence(videoInfo.license) let tags = [] @@ -186,7 +207,7 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: st const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo) - const videoAttributes = { + const defaultAttributes = { name: truncate(videoInfo.title, { 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max, 'separator': /,? +/, @@ -194,44 +215,46 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: st }), category, licence, - language, nsfw: isNSFW(videoInfo), - waitTranscoding: true, - commentsEnabled: true, - downloadEnabled: true, - description: videoInfo.description || undefined, - support: undefined, - tags, - privacy: VideoPrivacy.PUBLIC, - fixture: videoPath, + description: videoInfo.description, + tags + } + + const videoAttributes = await buildVideoAttributesFromCommander(url, program, defaultAttributes) + + Object.assign(videoAttributes, { + originallyPublishedAt: originallyPublishedAt ? originallyPublishedAt.toISOString() : null, thumbnailfile, previewfile: thumbnailfile, - originallyPublishedAt: originallyPublishedAt ? originallyPublishedAt.toISOString() : null - } + fixture: videoPath + }) + + log.info('\nUploading on PeerTube video "%s".', videoAttributes.name) + + let accessToken = await getAccessTokenOrDie(url, user) - console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) try { await uploadVideo(url, accessToken, videoAttributes) } catch (err) { if (err.message.indexOf('401') !== -1) { - console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.') + log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.') - const res = await login(url, client, user) - accessToken = res.body.access_token + accessToken = await getAccessTokenOrDie(url, user) await uploadVideo(url, accessToken, videoAttributes) } else { - console.log(err.message) - process.exit(1) + exitError(err.message) } } await remove(videoPath) if (thumbnailfile) await remove(thumbnailfile) - console.log('Uploaded video "%s"!\n', videoAttributes.name) + log.warn('Uploaded video "%s"!\n', videoAttributes.name) } +/* ---------------------------------------------------------- */ + async function getCategory (categories: string[], url: string) { if (!categories) return undefined @@ -250,8 +273,6 @@ async function getCategory (categories: string[], url: string) { return undefined } -/* ---------------------------------------------------------- */ - function getLicence (licence: string) { if (!licence) return undefined @@ -305,13 +326,66 @@ function buildUrl (info: any) { } function isNSFW (info: any) { - if (info.age_limit && info.age_limit >= 16) return true - - return false + return info.age_limit && info.age_limit >= 16 } function removeEndSlashes (url: string) { - while (url.endsWith('/')) { - url.slice(0, -1) + return url.replace(/\/+$/, '') +} + +async function promptPassword () { + return new Promise((res, rej) => { + prompt.start() + const schema = { + properties: { + password: { + hidden: true, + required: true + } + } + } + prompt.get(schema, function (err, result) { + if (err) { + return rej(err) + } + return res(result.password) + }) + }) +} + +async function getAccessTokenOrDie (url: string, user: UserInfo) { + const resClient = await getClient(url) + const client = { + id: resClient.body.client_id, + secret: resClient.body.client_secret } + + try { + const res = await login(url, client, user) + return res.body.access_token + } catch (err) { + exitError('Cannot authenticate. Please check your username/password.') + } +} + +function parseDate (dateAsStr: string): Date { + if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) { + exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`); + } + const date = new Date(dateAsStr); + date.setHours(0, 0, 0); + if (isNaN(date.getTime())) { + exitError(`Invalid date passed: ${dateAsStr}. See help for usage.`); + } + return date; +} + +function formatDate (date: Date): string { + return date.toISOString().split('T')[0]; +} + +function exitError (message:string, ...meta: any[]) { + // use console.error instead of log.error here + console.error(message, ...meta) + process.exit(-1) }