From 61b3e146e16e997ea539cd4610af10d4b681e04a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 20 Feb 2018 18:01:38 +0100 Subject: Add ability to import videos from all supported youtube-dl sites --- CHANGELOG.md | 17 ++ README.md | 2 +- .../videos/+video-edit/video-update.component.ts | 2 +- scripts/danger/clean/dev.sh | 2 +- scripts/danger/clean/prod.sh | 2 +- server/tools/import-videos.ts | 235 +++++++++++++++++++++ server/tools/import-youtube.ts | 201 ------------------ support/doc/import-videos.md | 63 ++++++ support/doc/import-youtube.md | 61 ------ 9 files changed, 319 insertions(+), 266 deletions(-) create mode 100644 server/tools/import-videos.ts delete mode 100644 server/tools/import-youtube.ts create mode 100644 support/doc/import-videos.md delete mode 100644 support/doc/import-youtube.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7541109..d28674c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ # Changelog +## v0.0.26-alpha + +### BREAKING CHANGES + + * Renamed script `import-youtube.js` to `import-videos.js` + * Renamed `import-video.js` argument `youtube-url` to `target-url` + +### Features + + * Add "Support" attribute/button on videos + * Add ability to import from all [supported sites](https://rg3.github.io/youtube-dl/supportedsites.html) of youtube-dl + +### Bug fixes + + * Fix custom instance name overflow + + ## v0.0.25-alpha ### Features diff --git a/README.md b/README.md index 5ff05fcb8..95f274b7b 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ For now only on Github: ## Tools - * [YouTube import](/support/doc/import-youtube.md) + * [Import videos (YouTube, Dailymotion, Vimeo...)](/support/doc/import-videos.md) ## Architecture diff --git a/client/src/app/videos/+video-edit/video-update.component.ts b/client/src/app/videos/+video-edit/video-update.component.ts index 0ef3c0259..d97e00a3a 100644 --- a/client/src/app/videos/+video-edit/video-update.component.ts +++ b/client/src/app/videos/+video-edit/video-update.component.ts @@ -61,7 +61,7 @@ export class VideoUpdateComponent extends FormReactive implements OnInit { .switchMap(video => { return this.videoService .loadCompleteDescription(video.descriptionPath) - .map(description => Object.assign(video, { description })) + .map(description => Object.assign(video, { description })) }) .subscribe( video => { diff --git a/scripts/danger/clean/dev.sh b/scripts/danger/clean/dev.sh index 270ca0a2e..cd8456772 100755 --- a/scripts/danger/clean/dev.sh +++ b/scripts/danger/clean/dev.sh @@ -4,5 +4,5 @@ read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then - NODE_ENV=test npm run ts-node "./scripts/danger/clean/cleaner" + NODE_ENV=test npm run ts-node -- --type-check "./scripts/danger/clean/cleaner" fi diff --git a/scripts/danger/clean/prod.sh b/scripts/danger/clean/prod.sh index 705987100..9103a7944 100755 --- a/scripts/danger/clean/prod.sh +++ b/scripts/danger/clean/prod.sh @@ -4,5 +4,5 @@ read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then - NODE_ENV=production npm run ts-node "./scripts/danger/clean/cleaner" + NODE_ENV=production npm run ts-node -- --type-check "./scripts/danger/clean/cleaner" fi diff --git a/server/tools/import-videos.ts b/server/tools/import-videos.ts new file mode 100644 index 000000000..268101b41 --- /dev/null +++ b/server/tools/import-videos.ts @@ -0,0 +1,235 @@ +import * as program from 'commander' +import { join } from 'path' +import * as youtubeDL from 'youtube-dl' +import { VideoPrivacy } from '../../shared/models/videos' +import { unlinkPromise } from '../helpers/core-utils' +import { doRequestAndSaveToFile } from '../helpers/requests' +import { CONSTRAINTS_FIELDS } from '../initializers' +import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils' + +program + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-t, --target-url ', 'Video target URL') + .option('-l, --language ', 'Language code') + .option('-v, --verbose', 'Verbose mode') + .parse(process.argv) + +if ( + !program['url'] || + !program['username'] || + !program['password'] || + !program['targetUrl'] +) { + console.error('All arguments are required.') + process.exit(-1) +} + +run().catch(err => console.error(err)) + +let accessToken: string +let client: { id: string, secret: string } + +const user = { + username: program['username'], + password: program['password'] +} + +const processOptions = { + cwd: __dirname, + maxBuffer: Infinity +} + +async function run () { + const res = await getClient(program['url']) + client = { + id: res.body.client_id, + secret: res.body.client_secret + } + + const res2 = await login(program['url'], client, user) + accessToken = res2.body.access_token + + const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] + youtubeDL.getInfo(program['targetUrl'], options, processOptions, async (err, info) => { + if (err) throw err + + let infoArray: any[] + + // Normalize utf8 fields + if (Array.isArray(info) === true) { + infoArray = info.map(i => normalizeObject(i)) + } else { + infoArray = [ normalizeObject(info) ] + } + console.log('Will download and upload %d videos.\n', infoArray.length) + + for (const info of infoArray) { + await processVideo(info, program['language']) + } + + // https://www.youtube.com/watch?v=2Upx39TBc1s + console.log('I\'m finished!') + process.exit(0) + }) +} + +function processVideo (info: any, languageCode: number) { + return new Promise(async res => { + if (program['verbose']) console.log('Fetching object.', info) + + const videoInfo = await fetchObject(info) + if (program['verbose']) console.log('Fetched object.', videoInfo) + + const result = await searchVideo(program['url'], videoInfo.title) + + console.log('############################################################\n') + + if (result.body.data.find(v => v.name === videoInfo.title)) { + console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title) + return res() + } + + const path = join(__dirname, new Date().getTime() + '.mp4') + + console.log('Downloading video "%s"...', videoInfo.title) + + const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] + youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => { + if (err) return console.error(err) + + console.log(output.join('\n')) + + await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, languageCode) + + return res() + }) + }) +} + +async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, language?: number) { + const category = await getCategory(videoInfo.categories) + const licence = getLicence(videoInfo.license) + let tags = [] + if (Array.isArray(videoInfo.tags)) { + tags = videoInfo.tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max) + .map(t => t.normalize()) + .slice(0, 5) + } + + let thumbnailfile + if (videoInfo.thumbnail) { + thumbnailfile = join(__dirname, 'thumbnail.jpg') + + await doRequestAndSaveToFile({ + method: 'GET', + uri: videoInfo.thumbnail + }, thumbnailfile) + } + + const videoAttributes = { + name: videoInfo.title, + category, + licence, + language, + nsfw: false, + commentsEnabled: true, + description: videoInfo.description, + tags, + privacy: VideoPrivacy.PUBLIC, + fixture: videoPath, + thumbnailfile, + previewfile: thumbnailfile + } + + console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) + try { + await uploadVideo(program['url'], accessToken, videoAttributes) + } catch (err) { + if (err.message.indexOf('401')) { + console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.') + + const res = await login(program['url'], client, user) + accessToken = res.body.access_token + + await uploadVideo(program['url'], accessToken, videoAttributes) + } else { + throw err + } + } + + await unlinkPromise(videoPath) + if (thumbnailfile) { + await unlinkPromise(thumbnailfile) + } + + console.log('Uploaded video "%s"!\n', videoAttributes.name) +} + +async function getCategory (categories: string[]) { + if (!categories) return undefined + + const categoryString = categories[0] + + if (categoryString === 'News & Politics') return 11 + + const res = await getVideoCategories(program['url']) + const categoriesServer = res.body + + for (const key of Object.keys(categoriesServer)) { + const categoryServer = categoriesServer[key] + if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10) + } + + return undefined +} + +function getLicence (licence: string) { + if (!licence) return undefined + + if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 + + return undefined +} + +function normalizeObject (obj: any) { + const newObj: any = {} + + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue + + const value = obj[key] + + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } + + return newObj +} + +function fetchObject (info: any) { + const url = buildUrl(info) + + return new Promise(async (res, rej) => { + youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => { + if (err) return rej(err) + + const videoInfoWithUrl = Object.assign(videoInfo, { url }) + return res(normalizeObject(videoInfoWithUrl)) + }) + }) +} + +function buildUrl (info: any) { + const url = info.url as string + if (url && url.match(/^https?:\/\//)) return info.url + + // It seems youtube-dl does not return the video url + return 'https://www.youtube.com/watch?v=' + info.id +} diff --git a/server/tools/import-youtube.ts b/server/tools/import-youtube.ts deleted file mode 100644 index 20b4b0179..000000000 --- a/server/tools/import-youtube.ts +++ /dev/null @@ -1,201 +0,0 @@ -import * as program from 'commander' -import { join } from 'path' -import * as youtubeDL from 'youtube-dl' -import { VideoPrivacy } from '../../shared/models/videos' -import { unlinkPromise } from '../helpers/core-utils' -import { doRequestAndSaveToFile } from '../helpers/requests' -import { CONSTRAINTS_FIELDS } from '../initializers' -import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils' - -program - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-y, --youtube-url ', 'Youtube URL') - .option('-l, --language ', 'Language code') - .parse(process.argv) - -if ( - !program['url'] || - !program['username'] || - !program['password'] || - !program['youtubeUrl'] -) { - console.error('All arguments are required.') - process.exit(-1) -} - -run().catch(err => console.error(err)) - -let accessToken: string -const processOptions = { - cwd: __dirname, - maxBuffer: Infinity -} - -async function run () { - const res = await getClient(program['url']) - const client = { - id: res.body.client_id, - secret: res.body.client_secret - } - - const user = { - username: program['username'], - password: program['password'] - } - - const res2 = await login(program['url'], client, user) - accessToken = res2.body.access_token - - const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] - youtubeDL.getInfo(program['youtubeUrl'], options, processOptions, async (err, info) => { - if (err) throw err - - // Normalize utf8 fields - info = info.map(i => normalizeObject(i)) - - const videos = info.map(i => { - return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title } - }) - - console.log('Will download and upload %d videos.\n', videos.length) - - for (const video of videos) { - await processVideo(video, program['language'], client, user) - } - - console.log('I have finished!') - process.exit(0) - }) -} - -function processVideo (video: { name: string, url: string }, languageCode: number, client: { id: string, secret: string }, user: { username: string, password: string }) { - return new Promise(async res => { - const result = await searchVideo(program['url'], video.name) - - console.log('############################################################\n') - - if (result.body.data.find(v => v.name === video.name)) { - console.log('Video "%s" already exists, don\'t reupload it.\n', video.name) - return res() - } - - const path = join(__dirname, new Date().getTime() + '.mp4') - - console.log('Downloading video "%s"...', video.name) - - const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ] - youtubeDL.exec(video.url, options, processOptions, async (err, output) => { - if (err) return console.error(err) - - console.log(output.join('\n')) - - youtubeDL.getInfo(video.url, undefined, processOptions, async (err, videoInfo) => { - if (err) return console.error(err) - - await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, client, user, languageCode) - - return res() - }) - }) - }) -} - -async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client: { id: string, secret: string }, user: { username: string, password: string }, language?: number) { - const category = await getCategory(videoInfo.categories) - const licence = getLicence(videoInfo.license) - let tags = [] - if (Array.isArray(videoInfo.tags)) { - tags = videoInfo.tags - .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max) - .map(t => t.normalize()) - .slice(0, 5) - } - - let thumbnailfile - if (videoInfo.thumbnail) { - thumbnailfile = join(__dirname, 'thumbnail.jpg') - - await doRequestAndSaveToFile({ - method: 'GET', - uri: videoInfo.thumbnail - }, thumbnailfile) - } - - const videoAttributes = { - name: videoInfo.title, - category, - licence, - language, - nsfw: false, - commentsEnabled: true, - description: videoInfo.description, - tags, - privacy: VideoPrivacy.PUBLIC, - fixture: videoPath, - thumbnailfile, - previewfile: thumbnailfile - } - - console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) - try { - await uploadVideo(program['url'], accessToken, videoAttributes) - } - catch (err) { - if ((err.message).search("401")) { - console.log("Get 401 Unauthorized, token may have expired, renewing token and retry.") - const res2 = await login(program['url'], client, user) - accessToken = res2.body.access_token - await uploadVideo(program['url'], accessToken, videoAttributes) - } - } - - await unlinkPromise(videoPath) - if (thumbnailfile) { - await unlinkPromise(thumbnailfile) - } - - console.log('Uploaded video "%s"!\n', videoAttributes.name) -} - -async function getCategory (categories: string[]) { - const categoryString = categories[0] - - if (categoryString === 'News & Politics') return 11 - - const res = await getVideoCategories(program['url']) - const categoriesServer = res.body - - for (const key of Object.keys(categoriesServer)) { - const categoryServer = categoriesServer[key] - if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10) - } - - return undefined -} - -function getLicence (licence: string) { - if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 - - return undefined -} - -function normalizeObject (obj: any) { - const newObj: any = {} - - for (const key of Object.keys(obj)) { - // Deprecated key - if (key === 'resolution') continue - - const value = obj[key] - - if (typeof value === 'string') { - newObj[key] = value.normalize() - } else { - newObj[key] = value - } - } - - return newObj -} diff --git a/support/doc/import-videos.md b/support/doc/import-videos.md new file mode 100644 index 000000000..166bb7c9f --- /dev/null +++ b/support/doc/import-videos.md @@ -0,0 +1,63 @@ +# Import videos guide + +You can use this script to import videos from all [supported sites of youtube-dl](https://rg3.github.io/youtube-dl/supportedsites.html) into PeerTube. +Be sure you own the videos or have the author's authorization to do so. + + - [Installation](#installation) + - [Usage](#usage) + +## Installation + +## Prerequisites + +You need at least 512MB RAM to run the script. +Importation can be launched directly from a PeerTube server (in this case you already have dependencies installed :+1:) or from a separate server, even a dekstop PC. + +### Dependencies + + * [PeerTube dependencies](dependencies.md) + +### Installation + +Clone the PeerTube repo to get the latest version: + +``` +git clone https://github.com/Chocobozzz/PeerTube.git +CLONE="$(pwd)/PeerTube" +``` + +Run ``yarn install`` +``` +cd ${CLONE} +yarn install +``` + +Build server tools: +``` +cd ${CLONE} +npm run build:server +``` + + +## Usage + +You are now ready to run the script : + +``` +cd ${CLONE} +node dist/server/tools/import-video.js -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD" -t "TARGET_URL" +``` + + * PEERTUBE_URL : the full URL of your PeerTube server where you want to import, eg: https://peertube.cpy.re/ + * PEERTUBE_USER : your PeerTube account where videos will be uploaded + * PEERTUBE_PASSWORD : password of your PeerTube account + * TARGET_URL : the target url you want to import. Examples: + * YouTube: + * Channel: https://www.youtube.com/channel/ChannelId + * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName + * Video https://www.youtube.com/watch?v=blabla + * Vimeo: https://vimeo.com/xxxxxx + * Dailymotion: https://www.dailymotion.com/xxxxx + + The script will get all public videos from Youtube, download them and upload to PeerTube. + Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection... diff --git a/support/doc/import-youtube.md b/support/doc/import-youtube.md deleted file mode 100644 index 39f01b85b..000000000 --- a/support/doc/import-youtube.md +++ /dev/null @@ -1,61 +0,0 @@ -# Import videos from Youtube guide - -You can use this script to import videos from Youtube channel to Peertube. -Be sure you own the videos or have the author's authorization to do so. - - - [Installation](#installation) - - [Usage](#usage) - -## Installation - -## Prerequisites - -You need at least 512MB RAM to run the script. -Importation can be launched directly from a PeerTube server (in this case you already have dependencies installed :+1:) or from a separate server, even a dekstop PC. - -### Dependencies - - * [PeerTube dependencies](dependencies.md) - * git - -### Installation - -Clone the PeerTube repo to get the latest version: - -``` -git clone https://github.com/Chocobozzz/PeerTube.git -CLONE="$(pwd)/PeerTube" -``` - -Run ``yarn install`` -``` -cd ${CLONE} -yarn install -``` - -Build server tools: -``` -cd ${CLONE} -npm run build:server -``` - - -## Usage - -You are now ready to run the script : - -``` -cd ${CLONE} -node dist/server/tools/import-youtube.js -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD" -y "YOUTUBE_URL" -``` - - * PEERTUBE_URL : the full URL of your PeerTube server where you want to import, eg: https://peertube.cpy.re/ - * PEERTUBE_USER : your PeerTube account where videos will be uploaded - * PEERTUBE_PASSWORD : password of your PeerTube account - * YOUTUBE_URL : the youtube video/user/channel/playlist you want to import. Examples: - * Channel: https://www.youtube.com/channel/ChannelId - * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName - * Video https://www.youtube.com/watch?v=blabla - - The script will get all public videos from Youtube, download them and upload to PeerTube. - Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection... -- cgit v1.2.3