aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-02 15:34:09 +0200
committerChocobozzz <me@florianbigard.com>2018-08-06 11:19:16 +0200
commitfbad87b0472f574409f7aa3ae7f8b54927d0cdd6 (patch)
tree197b4209e75d57dabae7cdd6f2da5f765e427023 /server/helpers
parent5e319fb7898fd0482c399cc3ae9dcfc20d274a58 (diff)
downloadPeerTube-fbad87b0472f574409f7aa3ae7f8b54927d0cdd6.tar.gz
PeerTube-fbad87b0472f574409f7aa3ae7f8b54927d0cdd6.tar.zst
PeerTube-fbad87b0472f574409f7aa3ae7f8b54927d0cdd6.zip
Add ability to import video with youtube-dl
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts2
-rw-r--r--server/helpers/custom-validators/video-imports.ts30
-rw-r--r--server/helpers/logger.ts2
-rw-r--r--server/helpers/youtube-dl.ts142
4 files changed, 174 insertions, 2 deletions
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index d97bbd2a9..c6a350236 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) {
45} 45}
46 46
47function sanitizeAndCheckVideoTorrentObject (video: any) { 47function sanitizeAndCheckVideoTorrentObject (video: any) {
48 if (video.type !== 'Video') return false 48 if (!video || video.type !== 'Video') return false
49 49
50 if (!setValidRemoteTags(video)) return false 50 if (!setValidRemoteTags(video)) return false
51 if (!setValidRemoteVideoUrls(video)) return false 51 if (!setValidRemoteVideoUrls(video)) return false
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
new file mode 100644
index 000000000..36c0559fd
--- /dev/null
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -0,0 +1,30 @@
1import 'express-validator'
2import 'multer'
3import * as validator from 'validator'
4import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
5import { exists } from './misc'
6
7function isVideoImportTargetUrlValid (url: string) {
8 const isURLOptions = {
9 require_host: true,
10 require_tld: true,
11 require_protocol: true,
12 require_valid_protocol: true,
13 protocols: [ 'http', 'https' ]
14 }
15
16 return exists(url) &&
17 validator.isURL('' + url, isURLOptions) &&
18 validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
19}
20
21function isVideoImportStateValid (value: any) {
22 return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined
23}
24
25// ---------------------------------------------------------------------------
26
27export {
28 isVideoImportStateValid,
29 isVideoImportTargetUrlValid
30}
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 04a19a9c6..480c5b49e 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) {
22} 22}
23 23
24const consoleLoggerFormat = winston.format.printf(info => { 24const consoleLoggerFormat = winston.format.printf(info => {
25 let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) 25 let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2)
26 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' 26 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
27 else additionalInfos = ' ' + additionalInfos 27 else additionalInfos = ' ' + additionalInfos
28 28
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
new file mode 100644
index 000000000..74d3e213b
--- /dev/null
+++ b/server/helpers/youtube-dl.ts
@@ -0,0 +1,142 @@
1import * as youtubeDL from 'youtube-dl'
2import { truncate } from 'lodash'
3import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
4import { join } from 'path'
5import * as crypto from 'crypto'
6import { logger } from './logger'
7
8export type YoutubeDLInfo = {
9 name: string
10 description: string
11 category: number
12 licence: number
13 nsfw: boolean
14 tags: string[]
15 thumbnailUrl: string
16}
17
18function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
19 return new Promise<YoutubeDLInfo>((res, rej) => {
20 const options = [ '-j', '--flat-playlist' ]
21
22 youtubeDL.getInfo(url, options, (err, info) => {
23 if (err) return rej(err)
24
25 const obj = normalizeObject(info)
26
27 return res(buildVideoInfo(obj))
28 })
29 })
30}
31
32function downloadYoutubeDLVideo (url: string) {
33 const hash = crypto.createHash('sha256').update(url).digest('base64')
34 const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
35
36 logger.info('Importing video %s', url)
37
38 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
39
40 return new Promise<string>((res, rej) => {
41 youtubeDL.exec(url, options, async (err, output) => {
42 if (err) return rej(err)
43
44 return res(path)
45 })
46 })
47}
48
49// ---------------------------------------------------------------------------
50
51export {
52 downloadYoutubeDLVideo,
53 getYoutubeDLInfo
54}
55
56// ---------------------------------------------------------------------------
57
58function normalizeObject (obj: any) {
59 const newObj: any = {}
60
61 for (const key of Object.keys(obj)) {
62 // Deprecated key
63 if (key === 'resolution') continue
64
65 const value = obj[key]
66
67 if (typeof value === 'string') {
68 newObj[key] = value.normalize()
69 } else {
70 newObj[key] = value
71 }
72 }
73
74 return newObj
75}
76
77function buildVideoInfo (obj: any) {
78 return {
79 name: titleTruncation(obj.title),
80 description: descriptionTruncation(obj.description),
81 category: getCategory(obj.categories),
82 licence: getLicence(obj.license),
83 nsfw: isNSFW(obj),
84 tags: getTags(obj.tags),
85 thumbnailUrl: obj.thumbnail || undefined
86 }
87}
88
89function titleTruncation (title: string) {
90 return truncate(title, {
91 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
92 'separator': /,? +/,
93 'omission': ' […]'
94 })
95}
96
97function descriptionTruncation (description: string) {
98 if (!description) return undefined
99
100 return truncate(description, {
101 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
102 'separator': /,? +/,
103 'omission': ' […]'
104 })
105}
106
107function isNSFW (info: any) {
108 return info.age_limit && info.age_limit >= 16
109}
110
111function getTags (tags: any) {
112 if (Array.isArray(tags) === false) return []
113
114 return tags
115 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
116 .map(t => t.normalize())
117 .slice(0, 5)
118}
119
120function getLicence (licence: string) {
121 if (!licence) return undefined
122
123 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
124
125 return undefined
126}
127
128function getCategory (categories: string[]) {
129 if (!categories) return undefined
130
131 const categoryString = categories[0]
132 if (!categoryString || typeof categoryString !== 'string') return undefined
133
134 if (categoryString === 'News & Politics') return 11
135
136 for (const key of Object.keys(VIDEO_CATEGORIES)) {
137 const category = VIDEO_CATEGORIES[key]
138 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
139 }
140
141 return undefined
142}