aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/tools
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-02-15 11:27:11 +0100
committerChocobozzz <me@florianbigard.com>2018-02-15 11:27:11 +0100
commit34cbef8c6cc912143a421413bdd832c4adcc556a (patch)
tree0620b5f3df06c682f64fb8ff5cfbeaf2f848eebe /server/tools
parent82815eb6700332b2fb41bcbbf2224471f66bb5a0 (diff)
downloadPeerTube-34cbef8c6cc912143a421413bdd832c4adcc556a.tar.gz
PeerTube-34cbef8c6cc912143a421413bdd832c4adcc556a.tar.zst
PeerTube-34cbef8c6cc912143a421413bdd832c4adcc556a.zip
Fix max buffer reached in youtube import
Diffstat (limited to 'server/tools')
-rw-r--r--server/tools/import-youtube.ts29
1 files changed, 20 insertions, 9 deletions
diff --git a/server/tools/import-youtube.ts b/server/tools/import-youtube.ts
index 12e996e5b..295109cd8 100644
--- a/server/tools/import-youtube.ts
+++ b/server/tools/import-youtube.ts
@@ -4,6 +4,7 @@ import * as youtubeDL from 'youtube-dl'
4import { VideoPrivacy } from '../../shared/models/videos' 4import { VideoPrivacy } from '../../shared/models/videos'
5import { unlinkPromise } from '../helpers/core-utils' 5import { unlinkPromise } from '../helpers/core-utils'
6import { doRequestAndSaveToFile } from '../helpers/requests' 6import { doRequestAndSaveToFile } from '../helpers/requests'
7import { CONSTRAINTS_FIELDS } from '../initializers'
7import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils' 8import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils'
8 9
9program 10program
@@ -11,6 +12,7 @@ program
11 .option('-U, --username <username>', 'Username') 12 .option('-U, --username <username>', 'Username')
12 .option('-p, --password <token>', 'Password') 13 .option('-p, --password <token>', 'Password')
13 .option('-y, --youtube-url <youtubeUrl>', 'Youtube URL') 14 .option('-y, --youtube-url <youtubeUrl>', 'Youtube URL')
15 .option('-l, --language <languageCode>', 'Language code')
14 .parse(process.argv) 16 .parse(process.argv)
15 17
16if ( 18if (
@@ -26,6 +28,10 @@ if (
26run().catch(err => console.error(err)) 28run().catch(err => console.error(err))
27 29
28let accessToken: string 30let accessToken: string
31const processOptions = {
32 cwd: __dirname,
33 maxBuffer: Infinity
34}
29 35
30async function run () { 36async function run () {
31 const res = await getClient(program['url']) 37 const res = await getClient(program['url'])
@@ -42,7 +48,8 @@ async function run () {
42 const res2 = await login(program['url'], client, user) 48 const res2 = await login(program['url'], client, user)
43 accessToken = res2.body.access_token 49 accessToken = res2.body.access_token
44 50
45 youtubeDL.getInfo(program['youtubeUrl'], [ '-j', '--flat-playlist' ], async (err, info) => { 51 const options = [ '-j', '--flat-playlist' ]
52 youtubeDL.getInfo(program['youtubeUrl'], options, processOptions, async (err, info) => {
46 if (err) throw err 53 if (err) throw err
47 54
48 // Normalize utf8 fields 55 // Normalize utf8 fields
@@ -55,7 +62,7 @@ async function run () {
55 console.log('Will download and upload %d videos.\n', videos.length) 62 console.log('Will download and upload %d videos.\n', videos.length)
56 63
57 for (const video of videos) { 64 for (const video of videos) {
58 await processVideo(video) 65 await processVideo(video, program['languageCode'])
59 } 66 }
60 67
61 console.log('I\'m finished!') 68 console.log('I\'m finished!')
@@ -63,7 +70,7 @@ async function run () {
63 }) 70 })
64} 71}
65 72
66function processVideo (video: { name: string, url: string }) { 73function processVideo (video: { name: string, url: string }, languageCode: number) {
67 return new Promise(async res => { 74 return new Promise(async res => {
68 const result = await searchVideo(program['url'], video.name) 75 const result = await searchVideo(program['url'], video.name)
69 76
@@ -78,15 +85,16 @@ function processVideo (video: { name: string, url: string }) {
78 85
79 console.log('Downloading video "%s"...', video.name) 86 console.log('Downloading video "%s"...', video.name)
80 87
81 youtubeDL.exec(video.url, [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ], {}, async (err, output) => { 88 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ]
89 youtubeDL.exec(video.url, options, processOptions, async (err, output) => {
82 if (err) return console.error(err) 90 if (err) return console.error(err)
83 91
84 console.log(output.join('\n')) 92 console.log(output.join('\n'))
85 93
86 youtubeDL.getInfo(video.url, async (err, videoInfo) => { 94 youtubeDL.getInfo(video.url, undefined, processOptions, async (err, videoInfo) => {
87 if (err) return console.error(err) 95 if (err) return console.error(err)
88 96
89 await uploadVideoOnPeerTube(normalizeObject(videoInfo), path) 97 await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, languageCode)
90 98
91 return res() 99 return res()
92 }) 100 })
@@ -94,10 +102,13 @@ function processVideo (video: { name: string, url: string }) {
94 }) 102 })
95} 103}
96 104
97async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string) { 105async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, language?: number) {
98 const category = await getCategory(videoInfo.categories) 106 const category = await getCategory(videoInfo.categories)
99 const licence = getLicence(videoInfo.license) 107 const licence = getLicence(videoInfo.license)
100 const language = 13 108 let tags = []
109 if (Array.isArray(videoInfo.tags)) {
110 tags = videoInfo.tags.filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max).slice(0, 5)
111 }
101 112
102 let thumbnailfile 113 let thumbnailfile
103 if (videoInfo.thumbnail) { 114 if (videoInfo.thumbnail) {
@@ -117,7 +128,7 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string) {
117 nsfw: false, 128 nsfw: false,
118 commentsEnabled: true, 129 commentsEnabled: true,
119 description: videoInfo.description, 130 description: videoInfo.description,
120 tags: videoInfo.tags.slice(0, 5), 131 tags,
121 privacy: VideoPrivacy.PUBLIC, 132 privacy: VideoPrivacy.PUBLIC,
122 fixture: videoPath, 133 fixture: videoPath,
123 thumbnailfile, 134 thumbnailfile,