aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-02-09 16:47:06 +0100
committerChocobozzz <me@florianbigard.com>2018-02-09 16:47:06 +0100
commita7fea183f0f69104b209e7bfdd6435be28165f22 (patch)
treea0bfb7daf8e9778c664b8d7224b2e1c382e3834e
parent1185c246c591a0783dc0ab268bbd67eba1d46bb9 (diff)
downloadPeerTube-a7fea183f0f69104b209e7bfdd6435be28165f22.tar.gz
PeerTube-a7fea183f0f69104b209e7bfdd6435be28165f22.tar.zst
PeerTube-a7fea183f0f69104b209e7bfdd6435be28165f22.zip
Begin import script with youtube-dl
-rw-r--r--package.json3
-rw-r--r--server/tests/utils/videos/videos.ts8
-rw-r--r--server/tools/get-access-token.ts (renamed from server/tests/real-world/tools/get-access-token.ts)2
-rw-r--r--server/tools/import-youtube.ts155
-rw-r--r--server/tools/upload-directory.ts (renamed from server/tests/real-world/tools/upload-directory.ts)4
-rw-r--r--server/tools/upload.ts (renamed from server/tests/real-world/tools/upload.ts)2
-rw-r--r--yarn.lock31
7 files changed, 198 insertions, 7 deletions
diff --git a/package.json b/package.json
index 948ba3f67..7ed520b65 100644
--- a/package.json
+++ b/package.json
@@ -135,7 +135,8 @@
135 "supertest": "^3.0.0", 135 "supertest": "^3.0.0",
136 "tslint": "^5.7.0", 136 "tslint": "^5.7.0",
137 "tslint-config-standard": "^7.0.0", 137 "tslint-config-standard": "^7.0.0",
138 "webtorrent": "^0.98.0" 138 "webtorrent": "^0.98.0",
139 "youtube-dl": "^1.12.2"
139 }, 140 },
140 "scripty": { 141 "scripty": {
141 "silent": true 142 "silent": true
diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts
index 0b28edd48..923ca48f1 100644
--- a/server/tests/utils/videos/videos.ts
+++ b/server/tests/utils/videos/videos.ts
@@ -249,8 +249,6 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
249 .set('Accept', 'application/json') 249 .set('Accept', 'application/json')
250 .set('Authorization', 'Bearer ' + accessToken) 250 .set('Authorization', 'Bearer ' + accessToken)
251 .field('name', attributes.name) 251 .field('name', attributes.name)
252 .field('category', attributes.category.toString())
253 .field('licence', attributes.licence.toString())
254 .field('nsfw', JSON.stringify(attributes.nsfw)) 252 .field('nsfw', JSON.stringify(attributes.nsfw))
255 .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) 253 .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled))
256 .field('description', attributes.description) 254 .field('description', attributes.description)
@@ -260,6 +258,12 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
260 if (attributes.language !== undefined) { 258 if (attributes.language !== undefined) {
261 req.field('language', attributes.language.toString()) 259 req.field('language', attributes.language.toString())
262 } 260 }
261 if (attributes.category !== undefined) {
262 req.field('category', attributes.category.toString())
263 }
264 if (attributes.licence !== undefined) {
265 req.field('licence', attributes.licence.toString())
266 }
263 267
264 for (let i = 0; i < attributes.tags.length; i++) { 268 for (let i = 0; i < attributes.tags.length; i++) {
265 req.field('tags[' + i + ']', attributes.tags[i]) 269 req.field('tags[' + i + ']', attributes.tags[i])
diff --git a/server/tests/real-world/tools/get-access-token.ts b/server/tools/get-access-token.ts
index ee14733e3..66fa70814 100644
--- a/server/tests/real-world/tools/get-access-token.ts
+++ b/server/tools/get-access-token.ts
@@ -3,7 +3,7 @@ import * as program from 'commander'
3import { 3import {
4 getClient, 4 getClient,
5 serverLogin 5 serverLogin
6} from '../../utils' 6} from '../tests/utils/index'
7 7
8program 8program
9 .option('-u, --url <url>', 'Server url') 9 .option('-u, --url <url>', 'Server url')
diff --git a/server/tools/import-youtube.ts b/server/tools/import-youtube.ts
new file mode 100644
index 000000000..b4405c452
--- /dev/null
+++ b/server/tools/import-youtube.ts
@@ -0,0 +1,155 @@
1import * as program from 'commander'
2import { createWriteStream } from 'fs'
3import { join } from 'path'
4import { cursorTo } from 'readline'
5import * as youtubeDL from 'youtube-dl'
6import { VideoPrivacy } from '../../shared/models/videos'
7import { unlinkPromise } from '../helpers/core-utils'
8import { getClient, getVideoCategories, login, searchVideo, uploadVideo } from '../tests/utils'
9
10program
11 .option('-u, --url <url>', 'Server url')
12 .option('-U, --username <username>', 'Username')
13 .option('-p, --password <token>', 'Password')
14 .option('-y, --youtube-url <directory>', 'Youtube URL')
15 .parse(process.argv)
16
17if (
18 !program['url'] ||
19 !program['username'] ||
20 !program['password'] ||
21 !program['youtubeUrl']
22) {
23 throw new Error('All arguments are required.')
24}
25
26run().catch(err => console.error(err))
27
28let accessToken: string
29
30async function run () {
31 const res = await getClient(program['url'])
32 const client = {
33 id: res.body.client_id,
34 secret: res.body.client_secret
35 }
36
37 const user = {
38 username: program['username'],
39 password: program['password']
40 }
41
42 const res2 = await login(program['url'], client, user)
43 accessToken = res2.body.access_token
44
45 youtubeDL.getInfo(program['youtubeUrl'], [ '-j', '--flat-playlist' ], async (err, info) => {
46 if (err) throw err
47
48 const videos = info.map(i => {
49 return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title }
50 })
51
52 console.log('Will download and upload %d videos.\n', videos.length)
53
54 for (const video of videos) {
55 await processVideo(video)
56 }
57
58 console.log('I\'m finished!')
59 process.exit(0)
60 })
61}
62
63function processVideo (videoUrlName: { name: string, url: string }) {
64 return new Promise(async res => {
65 const result = await searchVideo(program['url'], videoUrlName.name)
66 if (result.body.total !== 0) {
67 console.log('Video "%s" already exist, don\'t reupload it.\n', videoUrlName.name)
68 return res()
69 }
70
71 const video = youtubeDL(videoUrlName.url)
72 let videoInfo
73 let videoPath: string
74
75 video.on('error', err => console.error(err))
76
77 let size = 0
78 video.on('info', info => {
79 videoInfo = info
80 size = info.size
81
82 videoPath = join(__dirname, size + '.mp4')
83 console.log('Creating "%s" of video "%s".', videoPath, videoInfo.title)
84
85 video.pipe(createWriteStream(videoPath))
86 })
87
88 let pos = 0
89 video.on('data', chunk => {
90 pos += chunk.length
91 // `size` should not be 0 here.
92 if (size) {
93 const percent = (pos / size * 100).toFixed(2)
94 writeWaitingPercent(percent)
95 }
96 })
97
98 video.on('end', async () => {
99 await uploadVideoOnPeerTube(videoInfo, videoPath)
100
101 return res()
102 })
103 })
104}
105
106function writeWaitingPercent (p: string) {
107 cursorTo(process.stdout, 0)
108 process.stdout.write(`waiting ... ${p}%`)
109}
110
111async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string) {
112 const category = await getCategory(videoInfo.categories)
113 const licence = getLicence(videoInfo.license)
114 const language = 13
115
116 const videoAttributes = {
117 name: videoInfo.title,
118 category,
119 licence,
120 language,
121 nsfw: false,
122 commentsEnabled: true,
123 description: videoInfo.description,
124 tags: videoInfo.tags.slice(0, 5),
125 privacy: VideoPrivacy.PUBLIC,
126 fixture: videoPath
127 }
128
129 console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
130 await uploadVideo(program['url'], accessToken, videoAttributes)
131 await unlinkPromise(videoPath)
132 console.log('Uploaded video "%s"!\n', videoAttributes.name)
133}
134
135async function getCategory (categories: string[]) {
136 const categoryString = categories[0]
137
138 if (categoryString === 'News & Politics') return 11
139
140 const res = await getVideoCategories(program['url'])
141 const categoriesServer = res.body
142
143 for (const key of Object.keys(categoriesServer)) {
144 const categoryServer = categoriesServer[key]
145 if (categoryString.toLowerCase() === categoryServer.toLowerCase()) return parseInt(key, 10)
146 }
147
148 return undefined
149}
150
151function getLicence (licence: string) {
152 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
153
154 return undefined
155}
diff --git a/server/tests/real-world/tools/upload-directory.ts b/server/tools/upload-directory.ts
index fdd56857a..c0094f852 100644
--- a/server/tests/real-world/tools/upload-directory.ts
+++ b/server/tools/upload-directory.ts
@@ -2,8 +2,8 @@ import * as program from 'commander'
2import * as Promise from 'bluebird' 2import * as Promise from 'bluebird'
3import { isAbsolute, join } from 'path' 3import { isAbsolute, join } from 'path'
4 4
5import { readdirPromise } from '../../../helpers/core-utils' 5import { readdirPromise } from '../helpers/core-utils'
6import { execCLI } from '../../utils' 6import { execCLI } from '../tests/utils/index'
7 7
8program 8program
9 .option('-u, --url <url>', 'Server url') 9 .option('-u, --url <url>', 'Server url')
diff --git a/server/tests/real-world/tools/upload.ts b/server/tools/upload.ts
index 81bc0d415..db59bbdff 100644
--- a/server/tests/real-world/tools/upload.ts
+++ b/server/tools/upload.ts
@@ -5,7 +5,7 @@ import { promisify } from 'util'
5 5
6const accessPromise = promisify(access) 6const accessPromise = promisify(access)
7 7
8import { uploadVideo } from '../../utils' 8import { uploadVideo } from '../tests/utils/index'
9 9
10program 10program
11 .option('-u, --url <url>', 'Server url') 11 .option('-u, --url <url>', 'Server url')
diff --git a/yarn.lock b/yarn.lock
index 3a8b4cb30..9608266d0 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3089,6 +3089,12 @@ hash.js@^1.0.0:
3089 inherits "^2.0.3" 3089 inherits "^2.0.3"
3090 minimalistic-assert "^1.0.0" 3090 minimalistic-assert "^1.0.0"
3091 3091
3092hashish@~0.0.4:
3093 version "0.0.4"
3094 resolved "https://registry.yarnpkg.com/hashish/-/hashish-0.0.4.tgz#6d60bc6ffaf711b6afd60e426d077988014e6554"
3095 dependencies:
3096 traverse ">=0.2.4"
3097
3092hawk@3.1.3, hawk@~3.1.3: 3098hawk@3.1.3, hawk@~3.1.3:
3093 version "3.1.3" 3099 version "3.1.3"
3094 resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4" 3100 resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4"
@@ -3111,6 +3117,12 @@ he@1.1.1:
3111 version "1.1.1" 3117 version "1.1.1"
3112 resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" 3118 resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
3113 3119
3120hh-mm-ss@^1.2.0:
3121 version "1.2.0"
3122 resolved "https://registry.yarnpkg.com/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz#6d0f0b8280824a634cb1d1f20e0bc7bc8b689948"
3123 dependencies:
3124 zero-fill "^2.2.3"
3125
3114highlight.js@^9.1.0: 3126highlight.js@^9.1.0:
3115 version "9.12.0" 3127 version "9.12.0"
3116 resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e" 3128 resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e"
@@ -6370,6 +6382,12 @@ stream-with-known-length-to-buffer@^1.0.0:
6370 dependencies: 6382 dependencies:
6371 once "^1.3.3" 6383 once "^1.3.3"
6372 6384
6385streamify@^0.2.9:
6386 version "0.2.9"
6387 resolved "https://registry.yarnpkg.com/streamify/-/streamify-0.2.9.tgz#8938b14db491e2b6be4f8d99cc4133c9f0384f0b"
6388 dependencies:
6389 hashish "~0.0.4"
6390
6373streamsearch@0.1.2: 6391streamsearch@0.1.2:
6374 version "0.1.2" 6392 version "0.1.2"
6375 resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a" 6393 resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
@@ -6724,6 +6742,10 @@ trace@^1.1.0:
6724 dependencies: 6742 dependencies:
6725 stack-chain "~1.3.1" 6743 stack-chain "~1.3.1"
6726 6744
6745traverse@>=0.2.4:
6746 version "0.6.6"
6747 resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.6.6.tgz#cbdf560fd7b9af632502fed40f918c157ea97137"
6748
6727tree-kill@^1.1.0: 6749tree-kill@^1.1.0:
6728 version "1.2.0" 6750 version "1.2.0"
6729 resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36" 6751 resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36"
@@ -7403,6 +7425,15 @@ yn@^2.0.0:
7403 version "2.0.0" 7425 version "2.0.0"
7404 resolved "https://registry.yarnpkg.com/yn/-/yn-2.0.0.tgz#e5adabc8acf408f6385fc76495684c88e6af689a" 7426 resolved "https://registry.yarnpkg.com/yn/-/yn-2.0.0.tgz#e5adabc8acf408f6385fc76495684c88e6af689a"
7405 7427
7428youtube-dl@^1.12.2:
7429 version "1.12.2"
7430 resolved "https://registry.yarnpkg.com/youtube-dl/-/youtube-dl-1.12.2.tgz#11985268564c92b229f62b43d97374f86a605d1d"
7431 dependencies:
7432 hh-mm-ss "^1.2.0"
7433 mkdirp "^0.5.1"
7434 request "^2.83.0"
7435 streamify "^0.2.9"
7436
7406zero-fill@^2.2.3: 7437zero-fill@^2.2.3:
7407 version "2.2.3" 7438 version "2.2.3"
7408 resolved "https://registry.yarnpkg.com/zero-fill/-/zero-fill-2.2.3.tgz#a3def06ba5e39ae644850bb4ca2ad4112b4855e9" 7439 resolved "https://registry.yarnpkg.com/zero-fill/-/zero-fill-2.2.3.tgz#a3def06ba5e39ae644850bb4ca2ad4112b4855e9"