aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/tools/peertube-import-videos.ts
diff options
context:
space:
mode:
authorRigel Kent <sendmemail@rigelk.eu>2018-09-13 14:27:44 +0200
committerRigel Kent <sendmemail@rigelk.eu>2018-09-14 11:08:55 +0200
commit8704acf49efc770d73bf07c10468ed8c74d28a83 (patch)
treeffd46289fcf9a13ac4412b167e9f71dfb35753c5 /server/tools/peertube-import-videos.ts
parent1d9d9cfdcf3983e3fd89026bc4b5633a8abf5752 (diff)
downloadPeerTube-8704acf49efc770d73bf07c10468ed8c74d28a83.tar.gz
PeerTube-8704acf49efc770d73bf07c10468ed8c74d28a83.tar.zst
PeerTube-8704acf49efc770d73bf07c10468ed8c74d28a83.zip
one cli to unite them all
Ash nazg thrakatulûk agh burzum-ishi krimpatul - refactor import-videos to use the youtubeDL helper - add very basic tests for the cli
Diffstat (limited to 'server/tools/peertube-import-videos.ts')
-rw-r--r--server/tools/peertube-import-videos.ts323
1 files changed, 323 insertions, 0 deletions
diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
new file mode 100644
index 000000000..13090a028
--- /dev/null
+++ b/server/tools/peertube-import-videos.ts
@@ -0,0 +1,323 @@
1// FIXME: https://github.com/nodejs/node/pull/16853
2require('tls').DEFAULT_ECDH_CURVE = 'auto'
3
4import * as program from 'commander'
5import { join } from 'path'
6import { VideoPrivacy } from '../../shared/models/videos'
7import { doRequestAndSaveToFile } from '../helpers/requests'
8import { CONSTRAINTS_FIELDS } from '../initializers'
9import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../tests/utils'
10import { truncate } from 'lodash'
11import * as prompt from 'prompt'
12import { remove } from 'fs-extra'
13import { safeGetYoutubeDL } from '../helpers/youtube-dl'
14import { getSettings, netrc } from './cli'
15
16let accessToken: string
17let client: { id: string, secret: string }
18
19const processOptions = {
20 cwd: __dirname,
21 maxBuffer: Infinity
22}
23
24program
25 .name('import-videos')
26 .option('-u, --url <url>', 'Server url')
27 .option('-U, --username <username>', 'Username')
28 .option('-p, --password <token>', 'Password')
29 .option('-t, --target-url <targetUrl>', 'Video target URL')
30 .option('-l, --language <languageCode>', 'Language ISO 639 code (fr or en...)')
31 .option('-v, --verbose', 'Verbose mode')
32 .parse(process.argv)
33
34getSettings()
35.then(settings => {
36 if (
37 (!program['url'] ||
38 !program['username'] ||
39 !program['password']) &&
40 (settings.remotes.length === 0)
41 ) {
42 if (!program['url']) console.error('--url field is required.')
43 if (!program['username']) console.error('--username field is required.')
44 if (!program['password']) console.error('--password field is required.')
45 if (!program['targetUrl']) console.error('--targetUrl field is required.')
46 process.exit(-1)
47 }
48
49 if (
50 (!program['url'] ||
51 !program['username'] ||
52 !program['password']) &&
53 (settings.remotes.length > 0)
54 ) {
55 if (!program['url']) {
56 program['url'] = (settings.default !== -1) ?
57 settings.remotes[settings.default] :
58 settings.remotes[0]
59 }
60 if (!program['username']) program['username'] = netrc.machines[program['url']].login
61 if (!program['password']) program['password'] = netrc.machines[program['url']].password
62 }
63
64 if (
65 !program['targetUrl']
66 ) {
67 if (!program['targetUrl']) console.error('--targetUrl field is required.')
68 process.exit(-1)
69 }
70
71 const user = {
72 username: program['username'],
73 password: program['password']
74 }
75
76 run(user, program['url']).catch(err => console.error(err))
77})
78
79async function promptPassword () {
80 return new Promise((res, rej) => {
81 prompt.start()
82 const schema = {
83 properties: {
84 password: {
85 hidden: true,
86 required: true
87 }
88 }
89 }
90 prompt.get(schema, function (err, result) {
91 if (err) {
92 return rej(err)
93 }
94 return res(result.password)
95 })
96 })
97}
98
99async function run (user, url: string) {
100 if (!user.password) {
101 user.password = await promptPassword()
102 }
103
104 const res = await getClient(url)
105 client = {
106 id: res.body.client_id,
107 secret: res.body.client_secret
108 }
109
110 const res2 = await login(url, client, user)
111 accessToken = res2.body.access_token
112
113 const youtubeDL = await safeGetYoutubeDL()
114
115 const options = [ '-j', '--flat-playlist', '--playlist-reverse' ]
116 youtubeDL.getInfo(program['targetUrl'], options, processOptions, async (err, info) => {
117 if (err) {
118 console.log(err.message)
119 process.exit(1)
120 }
121
122 let infoArray: any[]
123
124 // Normalize utf8 fields
125 if (Array.isArray(info) === true) {
126 infoArray = info.map(i => normalizeObject(i))
127 } else {
128 infoArray = [ normalizeObject(info) ]
129 }
130 console.log('Will download and upload %d videos.\n', infoArray.length)
131
132 for (const info of infoArray) {
133 await processVideo(info, program['language'], processOptions.cwd, url, user)
134 }
135
136 // https://www.youtube.com/watch?v=2Upx39TBc1s
137 console.log('I\'m finished!')
138 process.exit(0)
139 })
140}
141
142function processVideo (info: any, languageCode: string, cwd: string, url: string, user) {
143 return new Promise(async res => {
144 if (program['verbose']) console.log('Fetching object.', info)
145
146 const videoInfo = await fetchObject(info)
147 if (program['verbose']) console.log('Fetched object.', videoInfo)
148
149 const result = await searchVideoWithSort(url, videoInfo.title, '-match')
150
151 console.log('############################################################\n')
152
153 if (result.body.data.find(v => v.name === videoInfo.title)) {
154 console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
155 return res()
156 }
157
158 const path = join(cwd, new Date().getTime() + '.mp4')
159
160 console.log('Downloading video "%s"...', videoInfo.title)
161
162 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
163 try {
164 const youtubeDL = await safeGetYoutubeDL()
165 youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
166 if (err) {
167 console.error(err)
168 return res()
169 }
170
171 console.log(output.join('\n'))
172 await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, cwd, url, user, languageCode)
173 return res()
174 })
175 } catch (err) {
176 console.log(err.message)
177 return res()
178 }
179 })
180}
181
182async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: string, url: string, user, language?: string) {
183 const category = await getCategory(videoInfo.categories, url)
184 const licence = getLicence(videoInfo.license)
185 let tags = []
186 if (Array.isArray(videoInfo.tags)) {
187 tags = videoInfo.tags
188 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
189 .map(t => t.normalize())
190 .slice(0, 5)
191 }
192
193 let thumbnailfile
194 if (videoInfo.thumbnail) {
195 thumbnailfile = join(cwd, 'thumbnail.jpg')
196
197 await doRequestAndSaveToFile({
198 method: 'GET',
199 uri: videoInfo.thumbnail
200 }, thumbnailfile)
201 }
202
203 const videoAttributes = {
204 name: truncate(videoInfo.title, {
205 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
206 'separator': /,? +/,
207 'omission': ' […]'
208 }),
209 category,
210 licence,
211 language,
212 nsfw: isNSFW(videoInfo),
213 waitTranscoding: true,
214 commentsEnabled: true,
215 description: videoInfo.description || undefined,
216 support: undefined,
217 tags,
218 privacy: VideoPrivacy.PUBLIC,
219 fixture: videoPath,
220 thumbnailfile,
221 previewfile: thumbnailfile
222 }
223
224 console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
225 try {
226 await uploadVideo(url, accessToken, videoAttributes)
227 } catch (err) {
228 if (err.message.indexOf('401') !== -1) {
229 console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.')
230
231 const res = await login(url, client, user)
232 accessToken = res.body.access_token
233
234 await uploadVideo(url, accessToken, videoAttributes)
235 } else {
236 console.log(err.message)
237 process.exit(1)
238 }
239 }
240
241 await remove(videoPath)
242 if (thumbnailfile) await remove(thumbnailfile)
243
244 console.log('Uploaded video "%s"!\n', videoAttributes.name)
245}
246
247async function getCategory (categories: string[], url: string) {
248 if (!categories) return undefined
249
250 const categoryString = categories[0]
251
252 if (categoryString === 'News & Politics') return 11
253
254 const res = await getVideoCategories(url)
255 const categoriesServer = res.body
256
257 for (const key of Object.keys(categoriesServer)) {
258 const categoryServer = categoriesServer[key]
259 if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10)
260 }
261
262 return undefined
263}
264
265/* ---------------------------------------------------------- */
266
267function getLicence (licence: string) {
268 if (!licence) return undefined
269
270 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
271
272 return undefined
273}
274
275function normalizeObject (obj: any) {
276 const newObj: any = {}
277
278 for (const key of Object.keys(obj)) {
279 // Deprecated key
280 if (key === 'resolution') continue
281
282 const value = obj[key]
283
284 if (typeof value === 'string') {
285 newObj[key] = value.normalize()
286 } else {
287 newObj[key] = value
288 }
289 }
290
291 return newObj
292}
293
294function fetchObject (info: any) {
295 const url = buildUrl(info)
296
297 return new Promise<any>(async (res, rej) => {
298 const youtubeDL = await safeGetYoutubeDL()
299 youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => {
300 if (err) return rej(err)
301
302 const videoInfoWithUrl = Object.assign(videoInfo, { url })
303 return res(normalizeObject(videoInfoWithUrl))
304 })
305 })
306}
307
308function buildUrl (info: any) {
309 const webpageUrl = info.webpage_url as string
310 if (webpageUrl && webpageUrl.match(/^https?:\/\//)) return webpageUrl
311
312 const url = info.url as string
313 if (url && url.match(/^https?:\/\//)) return url
314
315 // It seems youtube-dl does not return the video url
316 return 'https://www.youtube.com/watch?v=' + info.id
317}
318
319function isNSFW (info: any) {
320 if (info.age_limit && info.age_limit >= 16) return true
321
322 return false
323}