]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blame - server/helpers/youtube-dl.ts
emit more specific status codes on video upload (#3423)
[github/Chocobozzz/PeerTube.git] / server / helpers / youtube-dl.ts
CommitLineData
86ad0cde 1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
fbad87b0 2import { logger } from './logger'
6040f87d 3import { generateVideoImportTmpPath } from './utils'
606c946e 4import { join } from 'path'
687c6180
C
5import { peertubeTruncate, root } from './core-utils'
6import { ensureDir, remove, writeFile } from 'fs-extra'
606c946e
C
7import * as request from 'request'
8import { createWriteStream } from 'fs'
be7ca0c6 9import { CONFIG } from '@server/initializers/config'
f2eb23cd 10import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
fbad87b0
C
11
12export type YoutubeDLInfo = {
ce33919c
C
13 name?: string
14 description?: string
15 category?: number
86ad0cde 16 language?: string
ce33919c
C
17 licence?: number
18 nsfw?: boolean
19 tags?: string[]
20 thumbnailUrl?: string
d57d1d83 21 fileExt?: string
c74c9be9 22 originallyPublishedAt?: Date
fbad87b0
C
23}
24
50ad0a1c 25export type YoutubeDLSubs = {
652c6416
C
26 language: string
27 filename: string
50ad0a1c 28 path: string
29}[]
30
cc680494
C
31const processOptions = {
32 maxBuffer: 1024 * 1024 * 10 // 10MB
33}
34
8704acf4 35function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
a1587156 36 return new Promise<YoutubeDLInfo>((res, rej) => {
93905586 37 let args = opts || [ '-j', '--flat-playlist' ]
e0409585
C
38
39 if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
40 args.push('--force-ipv4')
41 }
42
93905586 43 args = wrapWithProxyOptions(args)
fbad87b0 44
a1587156
C
45 safeGetYoutubeDL()
46 .then(youtubeDL => {
47 youtubeDL.getInfo(url, args, processOptions, (err, info) => {
48 if (err) return rej(err)
49 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
fbad87b0 50
a1587156
C
51 const obj = buildVideoInfo(normalizeObject(info))
52 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
fbad87b0 53
a1587156
C
54 return res(obj)
55 })
56 })
57 .catch(err => rej(err))
fbad87b0
C
58 })
59}
60
50ad0a1c 61function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
62 return new Promise<YoutubeDLSubs>((res, rej) => {
63 const cwd = CONFIG.STORAGE.TMP_DIR
64 const options = opts || { all: true, format: 'vtt', cwd }
65
66 safeGetYoutubeDL()
67 .then(youtubeDL => {
68 youtubeDL.getSubs(url, options, (err, files) => {
69 if (err) return rej(err)
c7763edd 70 if (!files) return []
50ad0a1c 71
652c6416
C
72 logger.debug('Get subtitles from youtube dl.', { url, files })
73
50ad0a1c 74 const subtitles = files.reduce((acc, filename) => {
75 const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
982f2fc9 76 if (!matched || !matched[1]) return acc
50ad0a1c 77
982f2fc9
C
78 return [
79 ...acc,
80 {
81 language: matched[1],
82 path: join(cwd, filename),
83 filename
84 }
85 ]
50ad0a1c 86 }, [])
87
88 return res(subtitles)
89 })
90 })
91 .catch(err => rej(err))
92 })
93}
94
d57d1d83
C
95function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
96 const path = generateVideoImportTmpPath(url, extension)
cf9166cf 97 let timer
fbad87b0 98
d57d1d83 99 logger.info('Importing youtubeDL video %s to %s', url, path)
fbad87b0 100
be7ca0c6 101 let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
93905586 102 options = wrapWithProxyOptions(options)
fbad87b0 103
5322589d 104 if (process.env.FFMPEG_PATH) {
be7ca0c6 105 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
5322589d
C
106 }
107
a1587156
C
108 return new Promise<string>((res, rej) => {
109 safeGetYoutubeDL()
110 .then(youtubeDL => {
111 youtubeDL.exec(url, options, processOptions, err => {
112 clearTimeout(timer)
cf9166cf 113
a1587156
C
114 if (err) {
115 remove(path)
116 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
cf9166cf 117
a1587156
C
118 return rej(err)
119 }
fbad87b0 120
a1587156
C
121 return res(path)
122 })
cf9166cf 123
a1587156
C
124 timer = setTimeout(() => {
125 const err = new Error('YoutubeDL download timeout.')
cf9166cf 126
a1587156
C
127 remove(path)
128 .finally(() => rej(err))
129 .catch(err => {
130 logger.error('Cannot remove %s in youtubeDL timeout.', path, { err })
131 return rej(err)
132 })
133 }, timeout)
134 })
135 .catch(err => rej(err))
fbad87b0
C
136 })
137}
138
606c946e
C
139// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
140// We rewrote it to avoid sync calls
141async function updateYoutubeDLBinary () {
142 logger.info('Updating youtubeDL binary.')
143
144 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
145 const bin = join(binDirectory, 'youtube-dl')
146 const detailsPath = join(binDirectory, 'details')
44fb5297 147 const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl'
606c946e
C
148
149 await ensureDir(binDirectory)
150
151 return new Promise(res => {
152 request.get(url, { followRedirect: false }, (err, result) => {
153 if (err) {
154 logger.error('Cannot update youtube-dl.', { err })
155 return res()
156 }
157
f2eb23cd 158 if (result.statusCode !== HttpStatusCode.FOUND_302) {
606c946e
C
159 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
160 return res()
161 }
162
163 const url = result.headers.location
164 const downloadFile = request.get(url)
a1587156 165 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
606c946e
C
166
167 downloadFile.on('response', result => {
f2eb23cd 168 if (result.statusCode !== HttpStatusCode.OK_200) {
606c946e
C
169 logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
170 return res()
171 }
172
173 downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
174 })
175
176 downloadFile.on('error', err => {
177 logger.error('youtube-dl update error.', { err })
178 return res()
179 })
180
181 downloadFile.on('end', () => {
182 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
183 writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
184 if (err) {
185 logger.error('youtube-dl update error: cannot write details.', { err })
186 return res()
187 }
188
189 logger.info('youtube-dl updated to version %s.', newVersion)
190 return res()
191 })
192 })
193 })
194 })
195}
196
4f1f6f03
C
197async function safeGetYoutubeDL () {
198 let youtubeDL
199
200 try {
201 youtubeDL = require('youtube-dl')
202 } catch (e) {
203 // Download binary
606c946e 204 await updateYoutubeDLBinary()
4f1f6f03
C
205 youtubeDL = require('youtube-dl')
206 }
207
208 return youtubeDL
209}
210
c74c9be9
C
211function buildOriginallyPublishedAt (obj: any) {
212 let originallyPublishedAt: Date = null
213
214 const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
215 if (uploadDateMatcher) {
216 originallyPublishedAt = new Date()
217 originallyPublishedAt.setHours(0, 0, 0, 0)
218
219 const year = parseInt(uploadDateMatcher[1], 10)
220 // Month starts from 0
221 const month = parseInt(uploadDateMatcher[2], 10) - 1
222 const day = parseInt(uploadDateMatcher[3], 10)
223
224 originallyPublishedAt.setFullYear(year, month, day)
225 }
226
227 return originallyPublishedAt
228}
229
8704acf4
RK
230// ---------------------------------------------------------------------------
231
232export {
0491173a 233 updateYoutubeDLBinary,
8704acf4 234 downloadYoutubeDLVideo,
50ad0a1c 235 getYoutubeDLSubs,
8704acf4 236 getYoutubeDLInfo,
c74c9be9
C
237 safeGetYoutubeDL,
238 buildOriginallyPublishedAt
8704acf4
RK
239}
240
241// ---------------------------------------------------------------------------
242
fbad87b0
C
243function normalizeObject (obj: any) {
244 const newObj: any = {}
245
246 for (const key of Object.keys(obj)) {
247 // Deprecated key
248 if (key === 'resolution') continue
249
250 const value = obj[key]
251
252 if (typeof value === 'string') {
253 newObj[key] = value.normalize()
254 } else {
255 newObj[key] = value
256 }
257 }
258
259 return newObj
260}
261
86ad0cde 262function buildVideoInfo (obj: any): YoutubeDLInfo {
fbad87b0
C
263 return {
264 name: titleTruncation(obj.title),
265 description: descriptionTruncation(obj.description),
266 category: getCategory(obj.categories),
267 licence: getLicence(obj.license),
86ad0cde 268 language: getLanguage(obj.language),
fbad87b0
C
269 nsfw: isNSFW(obj),
270 tags: getTags(obj.tags),
4e553a41 271 thumbnailUrl: obj.thumbnail || undefined,
d57d1d83
C
272 originallyPublishedAt: buildOriginallyPublishedAt(obj),
273 fileExt: obj.ext
fbad87b0
C
274 }
275}
276
277function titleTruncation (title: string) {
687c6180
C
278 return peertubeTruncate(title, {
279 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
280 separator: /,? +/,
281 omission: ' […]'
fbad87b0
C
282 })
283}
284
285function descriptionTruncation (description: string) {
ed31c059 286 if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
fbad87b0 287
687c6180
C
288 return peertubeTruncate(description, {
289 length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
290 separator: /,? +/,
291 omission: ' […]'
fbad87b0
C
292 })
293}
294
295function isNSFW (info: any) {
296 return info.age_limit && info.age_limit >= 16
297}
298
299function getTags (tags: any) {
300 if (Array.isArray(tags) === false) return []
301
302 return tags
303 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
304 .map(t => t.normalize())
305 .slice(0, 5)
306}
307
308function getLicence (licence: string) {
309 if (!licence) return undefined
310
bdd428a6 311 if (licence.includes('Creative Commons Attribution')) return 1
fbad87b0 312
86ad0cde
C
313 for (const key of Object.keys(VIDEO_LICENCES)) {
314 const peertubeLicence = VIDEO_LICENCES[key]
315 if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
316 }
317
fbad87b0
C
318 return undefined
319}
320
321function getCategory (categories: string[]) {
322 if (!categories) return undefined
323
324 const categoryString = categories[0]
325 if (!categoryString || typeof categoryString !== 'string') return undefined
326
327 if (categoryString === 'News & Politics') return 11
328
329 for (const key of Object.keys(VIDEO_CATEGORIES)) {
330 const category = VIDEO_CATEGORIES[key]
331 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
332 }
333
334 return undefined
335}
93905586 336
86ad0cde
C
337function getLanguage (language: string) {
338 return VIDEO_LANGUAGES[language] ? language : undefined
339}
340
93905586
C
341function wrapWithProxyOptions (options: string[]) {
342 if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
343 logger.debug('Using proxy for YoutubeDL')
344
345 return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options)
346 }
347
348 return options
349}