aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-02-20 18:01:38 +0100
committerChocobozzz <me@florianbigard.com>2018-02-20 18:16:13 +0100
commit61b3e146e16e997ea539cd4610af10d4b681e04a (patch)
tree65937b83e8d01a6401b8cecd1fcf34de15aed1da
parent71578f317e881f35ec905e9136f77740bbd7e7aa (diff)
downloadPeerTube-61b3e146e16e997ea539cd4610af10d4b681e04a.tar.gz
PeerTube-61b3e146e16e997ea539cd4610af10d4b681e04a.tar.zst
PeerTube-61b3e146e16e997ea539cd4610af10d4b681e04a.zip
Add ability to import videos from all supported youtube-dl sites
-rw-r--r--CHANGELOG.md17
-rw-r--r--README.md2
-rw-r--r--client/src/app/videos/+video-edit/video-update.component.ts2
-rwxr-xr-xscripts/danger/clean/dev.sh2
-rwxr-xr-xscripts/danger/clean/prod.sh2
-rw-r--r--server/tools/import-videos.ts (renamed from server/tools/import-youtube.ts)112
-rw-r--r--support/doc/import-videos.md (renamed from support/doc/import-youtube.md)18
7 files changed, 104 insertions, 51 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce7541109..d28674c6d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,23 @@
1# Changelog 1# Changelog
2 2
3 3
4## v0.0.26-alpha
5
6### BREAKING CHANGES
7
8 * Renamed script `import-youtube.js` to `import-videos.js`
9 * Renamed `import-video.js` argument `youtube-url` to `target-url`
10
11### Features
12
13 * Add "Support" attribute/button on videos
14 * Add ability to import from all [supported sites](https://rg3.github.io/youtube-dl/supportedsites.html) of youtube-dl
15
16### Bug fixes
17
18 * Fix custom instance name overflow
19
20
4## v0.0.25-alpha 21## v0.0.25-alpha
5 22
6### Features 23### Features
diff --git a/README.md b/README.md
index 5ff05fcb8..95f274b7b 100644
--- a/README.md
+++ b/README.md
@@ -163,7 +163,7 @@ For now only on Github:
163 163
164## Tools 164## Tools
165 165
166 * [YouTube import](/support/doc/import-youtube.md) 166 * [Import videos (YouTube, Dailymotion, Vimeo...)](/support/doc/import-videos.md)
167 167
168## Architecture 168## Architecture
169 169
diff --git a/client/src/app/videos/+video-edit/video-update.component.ts b/client/src/app/videos/+video-edit/video-update.component.ts
index 0ef3c0259..d97e00a3a 100644
--- a/client/src/app/videos/+video-edit/video-update.component.ts
+++ b/client/src/app/videos/+video-edit/video-update.component.ts
@@ -61,7 +61,7 @@ export class VideoUpdateComponent extends FormReactive implements OnInit {
61 .switchMap(video => { 61 .switchMap(video => {
62 return this.videoService 62 return this.videoService
63 .loadCompleteDescription(video.descriptionPath) 63 .loadCompleteDescription(video.descriptionPath)
64 .map(description => Object.assign(video, { description })) 64 .map(description => Object.assign(video, { description }))
65 }) 65 })
66 .subscribe( 66 .subscribe(
67 video => { 67 video => {
diff --git a/scripts/danger/clean/dev.sh b/scripts/danger/clean/dev.sh
index 270ca0a2e..cd8456772 100755
--- a/scripts/danger/clean/dev.sh
+++ b/scripts/danger/clean/dev.sh
@@ -4,5 +4,5 @@ read -p "This will remove all directories and SQL tables. Are you sure? (y/*) "
4echo 4echo
5 5
6if [[ "$REPLY" =~ ^[Yy]$ ]]; then 6if [[ "$REPLY" =~ ^[Yy]$ ]]; then
7 NODE_ENV=test npm run ts-node "./scripts/danger/clean/cleaner" 7 NODE_ENV=test npm run ts-node -- --type-check "./scripts/danger/clean/cleaner"
8fi 8fi
diff --git a/scripts/danger/clean/prod.sh b/scripts/danger/clean/prod.sh
index 705987100..9103a7944 100755
--- a/scripts/danger/clean/prod.sh
+++ b/scripts/danger/clean/prod.sh
@@ -4,5 +4,5 @@ read -p "This will remove all directories and SQL tables. Are you sure? (y/*) "
4echo 4echo
5 5
6if [[ "$REPLY" =~ ^[Yy]$ ]]; then 6if [[ "$REPLY" =~ ^[Yy]$ ]]; then
7 NODE_ENV=production npm run ts-node "./scripts/danger/clean/cleaner" 7 NODE_ENV=production npm run ts-node -- --type-check "./scripts/danger/clean/cleaner"
8fi 8fi
diff --git a/server/tools/import-youtube.ts b/server/tools/import-videos.ts
index 20b4b0179..268101b41 100644
--- a/server/tools/import-youtube.ts
+++ b/server/tools/import-videos.ts
@@ -11,15 +11,16 @@ program
11 .option('-u, --url <url>', 'Server url') 11 .option('-u, --url <url>', 'Server url')
12 .option('-U, --username <username>', 'Username') 12 .option('-U, --username <username>', 'Username')
13 .option('-p, --password <token>', 'Password') 13 .option('-p, --password <token>', 'Password')
14 .option('-y, --youtube-url <youtubeUrl>', 'Youtube URL') 14 .option('-t, --target-url <targetUrl>', 'Video target URL')
15 .option('-l, --language <languageCode>', 'Language code') 15 .option('-l, --language <languageCode>', 'Language code')
16 .option('-v, --verbose', 'Verbose mode')
16 .parse(process.argv) 17 .parse(process.argv)
17 18
18if ( 19if (
19 !program['url'] || 20 !program['url'] ||
20 !program['username'] || 21 !program['username'] ||
21 !program['password'] || 22 !program['password'] ||
22 !program['youtubeUrl'] 23 !program['targetUrl']
23) { 24) {
24 console.error('All arguments are required.') 25 console.error('All arguments are required.')
25 process.exit(-1) 26 process.exit(-1)
@@ -28,6 +29,13 @@ if (
28run().catch(err => console.error(err)) 29run().catch(err => console.error(err))
29 30
30let accessToken: string 31let accessToken: string
32let client: { id: string, secret: string }
33
34const user = {
35 username: program['username'],
36 password: program['password']
37}
38
31const processOptions = { 39const processOptions = {
32 cwd: __dirname, 40 cwd: __dirname,
33 maxBuffer: Infinity 41 maxBuffer: Infinity
@@ -35,74 +43,72 @@ const processOptions = {
35 43
36async function run () { 44async function run () {
37 const res = await getClient(program['url']) 45 const res = await getClient(program['url'])
38 const client = { 46 client = {
39 id: res.body.client_id, 47 id: res.body.client_id,
40 secret: res.body.client_secret 48 secret: res.body.client_secret
41 } 49 }
42 50
43 const user = {
44 username: program['username'],
45 password: program['password']
46 }
47
48 const res2 = await login(program['url'], client, user) 51 const res2 = await login(program['url'], client, user)
49 accessToken = res2.body.access_token 52 accessToken = res2.body.access_token
50 53
51 const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] 54 const options = [ '-j', '--flat-playlist', '--playlist-reverse' ]
52 youtubeDL.getInfo(program['youtubeUrl'], options, processOptions, async (err, info) => { 55 youtubeDL.getInfo(program['targetUrl'], options, processOptions, async (err, info) => {
53 if (err) throw err 56 if (err) throw err
54 57
55 // Normalize utf8 fields 58 let infoArray: any[]
56 info = info.map(i => normalizeObject(i))
57
58 const videos = info.map(i => {
59 return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title }
60 })
61 59
62 console.log('Will download and upload %d videos.\n', videos.length) 60 // Normalize utf8 fields
61 if (Array.isArray(info) === true) {
62 infoArray = info.map(i => normalizeObject(i))
63 } else {
64 infoArray = [ normalizeObject(info) ]
65 }
66 console.log('Will download and upload %d videos.\n', infoArray.length)
63 67
64 for (const video of videos) { 68 for (const info of infoArray) {
65 await processVideo(video, program['language'], client, user) 69 await processVideo(info, program['language'])
66 } 70 }
67 71
68 console.log('I have finished!') 72 // https://www.youtube.com/watch?v=2Upx39TBc1s
73 console.log('I\'m finished!')
69 process.exit(0) 74 process.exit(0)
70 }) 75 })
71} 76}
72 77
73function processVideo (video: { name: string, url: string }, languageCode: number, client: { id: string, secret: string }, user: { username: string, password: string }) { 78function processVideo (info: any, languageCode: number) {
74 return new Promise(async res => { 79 return new Promise(async res => {
75 const result = await searchVideo(program['url'], video.name) 80 if (program['verbose']) console.log('Fetching object.', info)
81
82 const videoInfo = await fetchObject(info)
83 if (program['verbose']) console.log('Fetched object.', videoInfo)
84
85 const result = await searchVideo(program['url'], videoInfo.title)
76 86
77 console.log('############################################################\n') 87 console.log('############################################################\n')
78 88
79 if (result.body.data.find(v => v.name === video.name)) { 89 if (result.body.data.find(v => v.name === videoInfo.title)) {
80 console.log('Video "%s" already exists, don\'t reupload it.\n', video.name) 90 console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
81 return res() 91 return res()
82 } 92 }
83 93
84 const path = join(__dirname, new Date().getTime() + '.mp4') 94 const path = join(__dirname, new Date().getTime() + '.mp4')
85 95
86 console.log('Downloading video "%s"...', video.name) 96 console.log('Downloading video "%s"...', videoInfo.title)
87 97
88 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ] 98 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
89 youtubeDL.exec(video.url, options, processOptions, async (err, output) => { 99 youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => {
90 if (err) return console.error(err) 100 if (err) return console.error(err)
91 101
92 console.log(output.join('\n')) 102 console.log(output.join('\n'))
93 103
94 youtubeDL.getInfo(video.url, undefined, processOptions, async (err, videoInfo) => { 104 await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, languageCode)
95 if (err) return console.error(err)
96
97 await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, client, user, languageCode)
98 105
99 return res() 106 return res()
100 })
101 }) 107 })
102 }) 108 })
103} 109}
104 110
105async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client: { id: string, secret: string }, user: { username: string, password: string }, language?: number) { 111async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, language?: number) {
106 const category = await getCategory(videoInfo.categories) 112 const category = await getCategory(videoInfo.categories)
107 const licence = getLicence(videoInfo.license) 113 const licence = getLicence(videoInfo.license)
108 let tags = [] 114 let tags = []
@@ -141,13 +147,16 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client:
141 console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) 147 console.log('\nUploading on PeerTube video "%s".', videoAttributes.name)
142 try { 148 try {
143 await uploadVideo(program['url'], accessToken, videoAttributes) 149 await uploadVideo(program['url'], accessToken, videoAttributes)
144 } 150 } catch (err) {
145 catch (err) { 151 if (err.message.indexOf('401')) {
146 if ((err.message).search("401")) { 152 console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.')
147 console.log("Get 401 Unauthorized, token may have expired, renewing token and retry.") 153
148 const res2 = await login(program['url'], client, user) 154 const res = await login(program['url'], client, user)
149 accessToken = res2.body.access_token 155 accessToken = res.body.access_token
156
150 await uploadVideo(program['url'], accessToken, videoAttributes) 157 await uploadVideo(program['url'], accessToken, videoAttributes)
158 } else {
159 throw err
151 } 160 }
152 } 161 }
153 162
@@ -160,6 +169,8 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client:
160} 169}
161 170
162async function getCategory (categories: string[]) { 171async function getCategory (categories: string[]) {
172 if (!categories) return undefined
173
163 const categoryString = categories[0] 174 const categoryString = categories[0]
164 175
165 if (categoryString === 'News & Politics') return 11 176 if (categoryString === 'News & Politics') return 11
@@ -176,6 +187,8 @@ async function getCategory (categories: string[]) {
176} 187}
177 188
178function getLicence (licence: string) { 189function getLicence (licence: string) {
190 if (!licence) return undefined
191
179 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 192 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
180 193
181 return undefined 194 return undefined
@@ -199,3 +212,24 @@ function normalizeObject (obj: any) {
199 212
200 return newObj 213 return newObj
201} 214}
215
216function fetchObject (info: any) {
217 const url = buildUrl(info)
218
219 return new Promise<any>(async (res, rej) => {
220 youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => {
221 if (err) return rej(err)
222
223 const videoInfoWithUrl = Object.assign(videoInfo, { url })
224 return res(normalizeObject(videoInfoWithUrl))
225 })
226 })
227}
228
229function buildUrl (info: any) {
230 const url = info.url as string
231 if (url && url.match(/^https?:\/\//)) return info.url
232
233 // It seems youtube-dl does not return the video url
234 return 'https://www.youtube.com/watch?v=' + info.id
235}
diff --git a/support/doc/import-youtube.md b/support/doc/import-videos.md
index 39f01b85b..166bb7c9f 100644
--- a/support/doc/import-youtube.md
+++ b/support/doc/import-videos.md
@@ -1,6 +1,6 @@
1# Import videos from Youtube guide 1# Import videos guide
2 2
3You can use this script to import videos from Youtube channel to Peertube. 3You can use this script to import videos from all [supported sites of youtube-dl](https://rg3.github.io/youtube-dl/supportedsites.html) into PeerTube.
4Be sure you own the videos or have the author's authorization to do so. 4Be sure you own the videos or have the author's authorization to do so.
5 5
6 - [Installation](#installation) 6 - [Installation](#installation)
@@ -16,7 +16,6 @@ Importation can be launched directly from a PeerTube server (in this case you al
16### Dependencies 16### Dependencies
17 17
18 * [PeerTube dependencies](dependencies.md) 18 * [PeerTube dependencies](dependencies.md)
19 * git
20 19
21### Installation 20### Installation
22 21
@@ -46,16 +45,19 @@ You are now ready to run the script :
46 45
47``` 46```
48cd ${CLONE} 47cd ${CLONE}
49node dist/server/tools/import-youtube.js -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD" -y "YOUTUBE_URL" 48node dist/server/tools/import-video.js -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD" -t "TARGET_URL"
50``` 49```
51 50
52 * PEERTUBE_URL : the full URL of your PeerTube server where you want to import, eg: https://peertube.cpy.re/ 51 * PEERTUBE_URL : the full URL of your PeerTube server where you want to import, eg: https://peertube.cpy.re/
53 * PEERTUBE_USER : your PeerTube account where videos will be uploaded 52 * PEERTUBE_USER : your PeerTube account where videos will be uploaded
54 * PEERTUBE_PASSWORD : password of your PeerTube account 53 * PEERTUBE_PASSWORD : password of your PeerTube account
55 * YOUTUBE_URL : the youtube video/user/channel/playlist you want to import. Examples: 54 * TARGET_URL : the target url you want to import. Examples:
56 * Channel: https://www.youtube.com/channel/ChannelId 55 * YouTube:
57 * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName 56 * Channel: https://www.youtube.com/channel/ChannelId
58 * Video https://www.youtube.com/watch?v=blabla 57 * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName
58 * Video https://www.youtube.com/watch?v=blabla
59 * Vimeo: https://vimeo.com/xxxxxx
60 * Dailymotion: https://www.dailymotion.com/xxxxx
59 61
60 The script will get all public videos from Youtube, download them and upload to PeerTube. 62 The script will get all public videos from Youtube, download them and upload to PeerTube.
61 Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection... 63 Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection...