aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-02-09 16:47:06 +0100
committerChocobozzz <me@florianbigard.com>2018-02-09 16:47:06 +0100
commita7fea183f0f69104b209e7bfdd6435be28165f22 (patch)
treea0bfb7daf8e9778c664b8d7224b2e1c382e3834e /server
parent1185c246c591a0783dc0ab268bbd67eba1d46bb9 (diff)
downloadPeerTube-a7fea183f0f69104b209e7bfdd6435be28165f22.tar.gz
PeerTube-a7fea183f0f69104b209e7bfdd6435be28165f22.tar.zst
PeerTube-a7fea183f0f69104b209e7bfdd6435be28165f22.zip
Begin import script with youtube-dl
Diffstat (limited to 'server')
-rw-r--r--server/tests/utils/videos/videos.ts8
-rw-r--r--server/tools/get-access-token.ts (renamed from server/tests/real-world/tools/get-access-token.ts)2
-rw-r--r--server/tools/import-youtube.ts155
-rw-r--r--server/tools/upload-directory.ts (renamed from server/tests/real-world/tools/upload-directory.ts)4
-rw-r--r--server/tools/upload.ts (renamed from server/tests/real-world/tools/upload.ts)2
5 files changed, 165 insertions, 6 deletions
diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts
index 0b28edd48..923ca48f1 100644
--- a/server/tests/utils/videos/videos.ts
+++ b/server/tests/utils/videos/videos.ts
@@ -249,8 +249,6 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
249 .set('Accept', 'application/json') 249 .set('Accept', 'application/json')
250 .set('Authorization', 'Bearer ' + accessToken) 250 .set('Authorization', 'Bearer ' + accessToken)
251 .field('name', attributes.name) 251 .field('name', attributes.name)
252 .field('category', attributes.category.toString())
253 .field('licence', attributes.licence.toString())
254 .field('nsfw', JSON.stringify(attributes.nsfw)) 252 .field('nsfw', JSON.stringify(attributes.nsfw))
255 .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) 253 .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled))
256 .field('description', attributes.description) 254 .field('description', attributes.description)
@@ -260,6 +258,12 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
260 if (attributes.language !== undefined) { 258 if (attributes.language !== undefined) {
261 req.field('language', attributes.language.toString()) 259 req.field('language', attributes.language.toString())
262 } 260 }
261 if (attributes.category !== undefined) {
262 req.field('category', attributes.category.toString())
263 }
264 if (attributes.licence !== undefined) {
265 req.field('licence', attributes.licence.toString())
266 }
263 267
264 for (let i = 0; i < attributes.tags.length; i++) { 268 for (let i = 0; i < attributes.tags.length; i++) {
265 req.field('tags[' + i + ']', attributes.tags[i]) 269 req.field('tags[' + i + ']', attributes.tags[i])
diff --git a/server/tests/real-world/tools/get-access-token.ts b/server/tools/get-access-token.ts
index ee14733e3..66fa70814 100644
--- a/server/tests/real-world/tools/get-access-token.ts
+++ b/server/tools/get-access-token.ts
@@ -3,7 +3,7 @@ import * as program from 'commander'
3import { 3import {
4 getClient, 4 getClient,
5 serverLogin 5 serverLogin
6} from '../../utils' 6} from '../tests/utils/index'
7 7
8program 8program
9 .option('-u, --url <url>', 'Server url') 9 .option('-u, --url <url>', 'Server url')
diff --git a/server/tools/import-youtube.ts b/server/tools/import-youtube.ts
new file mode 100644
index 000000000..b4405c452
--- /dev/null
+++ b/server/tools/import-youtube.ts
@@ -0,0 +1,155 @@
1import * as program from 'commander'
2import { createWriteStream } from 'fs'
3import { join } from 'path'
4import { cursorTo } from 'readline'
5import * as youtubeDL from 'youtube-dl'
6import { VideoPrivacy } from '../../shared/models/videos'
7import { unlinkPromise } from '../helpers/core-utils'
8import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils'
9
10program
11 .option('-u, --url <url>', 'Server url')
12 .option('-U, --username <username>', 'Username')
13 .option('-p, --password <token>', 'Password')
14 .option('-y, --youtube-url <directory>', 'Youtube URL')
15 .parse(process.argv)
16
17if (
18 !program['url'] ||
19 !program['username'] ||
20 !program['password'] ||
21 !program['youtubeUrl']
22) {
23 throw new Error('All arguments are required.')
24}
25
26run().catch(err => console.error(err))
27
28let accessToken: string
29
30async function run () {
31 const res = await getClient(program['url'])
32 const client = {
33 id: res.body.client_id,
34 secret: res.body.client_secret
35 }
36
37 const user = {
38 username: program['username'],
39 password: program['password']
40 }
41
42 const res2 = await login(program['url'], client, user)
43 accessToken = res2.body.access_token
44
45 youtubeDL.getInfo(program['youtubeUrl'], [ '-j', '--flat-playlist' ], async (err, info) => {
46 if (err) throw err
47
48 const videos = info.map(i => {
49 return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title }
50 })
51
52 console.log('Will download and upload %d videos.\n', videos.length)
53
54 for (const video of videos) {
55 await processVideo(video)
56 }
57
58 console.log('I\'m finished!')
59 process.exit(0)
60 })
61}
62
63function processVideo (videoUrlName: { name: string, url: string }) {
64 return new Promise(async res => {
65 const result = await searchVideo(program['url'], videoUrlName.name)
66 if (result.body.total !== 0) {
67 console.log('Video "%s" already exist, don\'t reupload it.\n', videoUrlName.name)
68 return res()
69 }
70
71 const video = youtubeDL(videoUrlName.url)
72 let videoInfo
73 let videoPath: string
74
75 video.on('error', err => console.error(err))
76
77 let size = 0
78 video.on('info', info => {
79 videoInfo = info
80 size = info.size
81
82 videoPath = join(__dirname, size + '.mp4')
83 console.log('Creating "%s" of video "%s".', videoPath, videoInfo.title)
84
85 video.pipe(createWriteStream(videoPath))
86 })
87
88 let pos = 0
89 video.on('data', chunk => {
90 pos += chunk.length
91 // `size` should not be 0 here.
92 if (size) {
93 const percent = (pos / size * 100).toFixed(2)
94 writeWaitingPercent(percent)
95 }
96 })
97
98 video.on('end', async () => {
99 await uploadVideoOnPeerTube(videoInfo, videoPath)
100
101 return res()
102 })
103 })
104}
105
106function writeWaitingPercent (p: string) {
107 cursorTo(process.stdout, 0)
108 process.stdout.write(`waiting ... ${p}%`)
109}
110
111async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string) {
112 const category = await getCategory(videoInfo.categories)
113 const licence = getLicence(videoInfo.license)
114 const language = 13
115
116 const videoAttributes = {
117 name: videoInfo.title,
118 category,
119 licence,
120 language,
121 nsfw: false,
122 commentsEnabled: true,
123 description: videoInfo.description,
124 tags: videoInfo.tags.slice(0, 5),
125 privacy: VideoPrivacy.PUBLIC,
126 fixture: videoPath
127 }
128
129 console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
130 await uploadVideo(program['url'], accessToken, videoAttributes)
131 await unlinkPromise(videoPath)
132 console.log('Uploaded video "%s"!\n', videoAttributes.name)
133}
134
135async function getCategory (categories: string[]) {
136 const categoryString = categories[0]
137
138 if (categoryString === 'News & Politics') return 11
139
140 const res = await getVideoCategories(program['url'])
141 const categoriesServer = res.body
142
143 for (const key of Object.keys(categoriesServer)) {
144 const categoryServer = categoriesServer[key]
145 if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10)
146 }
147
148 return undefined
149}
150
151function getLicence (licence: string) {
152 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
153
154 return undefined
155}
diff --git a/server/tests/real-world/tools/upload-directory.ts b/server/tools/upload-directory.ts
index fdd56857a..c0094f852 100644
--- a/server/tests/real-world/tools/upload-directory.ts
+++ b/server/tools/upload-directory.ts
@@ -2,8 +2,8 @@ import * as program from 'commander'
2import * as Promise from 'bluebird' 2import * as Promise from 'bluebird'
3import { isAbsolute, join } from 'path' 3import { isAbsolute, join } from 'path'
4 4
5import { readdirPromise } from '../../../helpers/core-utils' 5import { readdirPromise } from '../helpers/core-utils'
6import { execCLI } from '../../utils' 6import { execCLI } from '../tests/utils/index'
7 7
8program 8program
9 .option('-u, --url <url>', 'Server url') 9 .option('-u, --url <url>', 'Server url')
diff --git a/server/tests/real-world/tools/upload.ts b/server/tools/upload.ts
index 81bc0d415..db59bbdff 100644
--- a/server/tests/real-world/tools/upload.ts
+++ b/server/tools/upload.ts
@@ -5,7 +5,7 @@ import { promisify } from 'util'
5 5
6const accessPromise = promisify(access) 6const accessPromise = promisify(access)
7 7
8import { uploadVideo } from '../../utils' 8import { uploadVideo } from '../tests/utils/index'
9 9
10program 10program
11 .option('-u, --url <url>', 'Server url') 11 .option('-u, --url <url>', 'Server url')