From 61b3e146e16e997ea539cd4610af10d4b681e04a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 20 Feb 2018 18:01:38 +0100 Subject: Add ability to import videos from all supported youtube-dl sites --- server/tools/import-videos.ts | 235 +++++++++++++++++++++++++++++++++++++++++ server/tools/import-youtube.ts | 201 ----------------------------------- 2 files changed, 235 insertions(+), 201 deletions(-) create mode 100644 server/tools/import-videos.ts delete mode 100644 server/tools/import-youtube.ts (limited to 'server') diff --git a/server/tools/import-videos.ts b/server/tools/import-videos.ts new file mode 100644 index 000000000..268101b41 --- /dev/null +++ b/server/tools/import-videos.ts @@ -0,0 +1,235 @@ +import * as program from 'commander' +import { join } from 'path' +import * as youtubeDL from 'youtube-dl' +import { VideoPrivacy } from '../../shared/models/videos' +import { unlinkPromise } from '../helpers/core-utils' +import { doRequestAndSaveToFile } from '../helpers/requests' +import { CONSTRAINTS_FIELDS } from '../initializers' +import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils' + +program + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-t, --target-url ', 'Video target URL') + .option('-l, --language ', 'Language code') + .option('-v, --verbose', 'Verbose mode') + .parse(process.argv) + +if ( + !program['url'] || + !program['username'] || + !program['password'] || + !program['targetUrl'] +) { + console.error('All arguments are required.') + process.exit(-1) +} + +run().catch(err => console.error(err)) + +let accessToken: string +let client: { id: string, secret: string } + +const user = { + username: program['username'], + password: program['password'] +} + +const processOptions = { + cwd: __dirname, + maxBuffer: Infinity +} + +async function run () { + const res = await getClient(program['url']) + client = { + id: res.body.client_id, + secret: res.body.client_secret + } + + const res2 = await login(program['url'], client, user) + accessToken = res2.body.access_token + + const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] + youtubeDL.getInfo(program['targetUrl'], options, processOptions, async (err, info) => { + if (err) throw err + + let infoArray: any[] + + // Normalize utf8 fields + if (Array.isArray(info) === true) { + infoArray = info.map(i => normalizeObject(i)) + } else { + infoArray = [ normalizeObject(info) ] + } + console.log('Will download and upload %d videos.\n', infoArray.length) + + for (const info of infoArray) { + await processVideo(info, program['language']) + } + + // https://www.youtube.com/watch?v=2Upx39TBc1s + console.log('I\'m finished!') + process.exit(0) + }) +} + +function processVideo (info: any, languageCode: number) { + return new Promise(async res => { + if (program['verbose']) console.log('Fetching object.', info) + + const videoInfo = await fetchObject(info) + if (program['verbose']) console.log('Fetched object.', videoInfo) + + const result = await searchVideo(program['url'], videoInfo.title) + + console.log('############################################################\n') + + if (result.body.data.find(v => v.name === videoInfo.title)) { + console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title) + return res() + } + + const path = join(__dirname, new Date().getTime() + '.mp4') + + console.log('Downloading video "%s"...', videoInfo.title) + + const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] + youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => { + if (err) return console.error(err) + + console.log(output.join('\n')) + + await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, languageCode) + + return res() + }) + }) +} + +async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, language?: number) { + const category = await getCategory(videoInfo.categories) + const licence = getLicence(videoInfo.license) + let tags = [] + if (Array.isArray(videoInfo.tags)) { + tags = videoInfo.tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max) + .map(t => t.normalize()) + .slice(0, 5) + } + + let thumbnailfile + if (videoInfo.thumbnail) { + thumbnailfile = join(__dirname, 'thumbnail.jpg') + + await doRequestAndSaveToFile({ + method: 'GET', + uri: videoInfo.thumbnail + }, thumbnailfile) + } + + const videoAttributes = { + name: videoInfo.title, + category, + licence, + language, + nsfw: false, + commentsEnabled: true, + description: videoInfo.description, + tags, + privacy: VideoPrivacy.PUBLIC, + fixture: videoPath, + thumbnailfile, + previewfile: thumbnailfile + } + + console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) + try { + await uploadVideo(program['url'], accessToken, videoAttributes) + } catch (err) { + if (err.message.indexOf('401')) { + console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.') + + const res = await login(program['url'], client, user) + accessToken = res.body.access_token + + await uploadVideo(program['url'], accessToken, videoAttributes) + } else { + throw err + } + } + + await unlinkPromise(videoPath) + if (thumbnailfile) { + await unlinkPromise(thumbnailfile) + } + + console.log('Uploaded video "%s"!\n', videoAttributes.name) +} + +async function getCategory (categories: string[]) { + if (!categories) return undefined + + const categoryString = categories[0] + + if (categoryString === 'News & Politics') return 11 + + const res = await getVideoCategories(program['url']) + const categoriesServer = res.body + + for (const key of Object.keys(categoriesServer)) { + const categoryServer = categoriesServer[key] + if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10) + } + + return undefined +} + +function getLicence (licence: string) { + if (!licence) return undefined + + if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 + + return undefined +} + +function normalizeObject (obj: any) { + const newObj: any = {} + + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue + + const value = obj[key] + + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } + + return newObj +} + +function fetchObject (info: any) { + const url = buildUrl(info) + + return new Promise(async (res, rej) => { + youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => { + if (err) return rej(err) + + const videoInfoWithUrl = Object.assign(videoInfo, { url }) + return res(normalizeObject(videoInfoWithUrl)) + }) + }) +} + +function buildUrl (info: any) { + const url = info.url as string + if (url && url.match(/^https?:\/\//)) return info.url + + // It seems youtube-dl does not return the video url + return 'https://www.youtube.com/watch?v=' + info.id +} diff --git a/server/tools/import-youtube.ts b/server/tools/import-youtube.ts deleted file mode 100644 index 20b4b0179..000000000 --- a/server/tools/import-youtube.ts +++ /dev/null @@ -1,201 +0,0 @@ -import * as program from 'commander' -import { join } from 'path' -import * as youtubeDL from 'youtube-dl' -import { VideoPrivacy } from '../../shared/models/videos' -import { unlinkPromise } from '../helpers/core-utils' -import { doRequestAndSaveToFile } from '../helpers/requests' -import { CONSTRAINTS_FIELDS } from '../initializers' -import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils' - -program - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-y, --youtube-url ', 'Youtube URL') - .option('-l, --language ', 'Language code') - .parse(process.argv) - -if ( - !program['url'] || - !program['username'] || - !program['password'] || - !program['youtubeUrl'] -) { - console.error('All arguments are required.') - process.exit(-1) -} - -run().catch(err => console.error(err)) - -let accessToken: string -const processOptions = { - cwd: __dirname, - maxBuffer: Infinity -} - -async function run () { - const res = await getClient(program['url']) - const client = { - id: res.body.client_id, - secret: res.body.client_secret - } - - const user = { - username: program['username'], - password: program['password'] - } - - const res2 = await login(program['url'], client, user) - accessToken = res2.body.access_token - - const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] - youtubeDL.getInfo(program['youtubeUrl'], options, processOptions, async (err, info) => { - if (err) throw err - - // Normalize utf8 fields - info = info.map(i => normalizeObject(i)) - - const videos = info.map(i => { - return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title } - }) - - console.log('Will download and upload %d videos.\n', videos.length) - - for (const video of videos) { - await processVideo(video, program['language'], client, user) - } - - console.log('I have finished!') - process.exit(0) - }) -} - -function processVideo (video: { name: string, url: string }, languageCode: number, client: { id: string, secret: string }, user: { username: string, password: string }) { - return new Promise(async res => { - const result = await searchVideo(program['url'], video.name) - - console.log('############################################################\n') - - if (result.body.data.find(v => v.name === video.name)) { - console.log('Video "%s" already exists, don\'t reupload it.\n', video.name) - return res() - } - - const path = join(__dirname, new Date().getTime() + '.mp4') - - console.log('Downloading video "%s"...', video.name) - - const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ] - youtubeDL.exec(video.url, options, processOptions, async (err, output) => { - if (err) return console.error(err) - - console.log(output.join('\n')) - - youtubeDL.getInfo(video.url, undefined, processOptions, async (err, videoInfo) => { - if (err) return console.error(err) - - await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, client, user, languageCode) - - return res() - }) - }) - }) -} - -async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client: { id: string, secret: string }, user: { username: string, password: string }, language?: number) { - const category = await getCategory(videoInfo.categories) - const licence = getLicence(videoInfo.license) - let tags = [] - if (Array.isArray(videoInfo.tags)) { - tags = videoInfo.tags - .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max) - .map(t => t.normalize()) - .slice(0, 5) - } - - let thumbnailfile - if (videoInfo.thumbnail) { - thumbnailfile = join(__dirname, 'thumbnail.jpg') - - await doRequestAndSaveToFile({ - method: 'GET', - uri: videoInfo.thumbnail - }, thumbnailfile) - } - - const videoAttributes = { - name: videoInfo.title, - category, - licence, - language, - nsfw: false, - commentsEnabled: true, - description: videoInfo.description, - tags, - privacy: VideoPrivacy.PUBLIC, - fixture: videoPath, - thumbnailfile, - previewfile: thumbnailfile - } - - console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) - try { - await uploadVideo(program['url'], accessToken, videoAttributes) - } - catch (err) { - if ((err.message).search("401")) { - console.log("Get 401 Unauthorized, token may have expired, renewing token and retry.") - const res2 = await login(program['url'], client, user) - accessToken = res2.body.access_token - await uploadVideo(program['url'], accessToken, videoAttributes) - } - } - - await unlinkPromise(videoPath) - if (thumbnailfile) { - await unlinkPromise(thumbnailfile) - } - - console.log('Uploaded video "%s"!\n', videoAttributes.name) -} - -async function getCategory (categories: string[]) { - const categoryString = categories[0] - - if (categoryString === 'News & Politics') return 11 - - const res = await getVideoCategories(program['url']) - const categoriesServer = res.body - - for (const key of Object.keys(categoriesServer)) { - const categoryServer = categoriesServer[key] - if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10) - } - - return undefined -} - -function getLicence (licence: string) { - if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 - - return undefined -} - -function normalizeObject (obj: any) { - const newObj: any = {} - - for (const key of Object.keys(obj)) { - // Deprecated key - if (key === 'resolution') continue - - const value = obj[key] - - if (typeof value === 'string') { - newObj[key] = value.normalize() - } else { - newObj[key] = value - } - } - - return newObj -} -- cgit v1.2.3