From fbad87b0472f574409f7aa3ae7f8b54927d0cdd6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 2 Aug 2018 15:34:09 +0200 Subject: Add ability to import video with youtube-dl --- .../custom-validators/activitypub/videos.ts | 2 +- server/helpers/custom-validators/video-imports.ts | 30 +++++ server/helpers/logger.ts | 2 +- server/helpers/youtube-dl.ts | 142 +++++++++++++++++++++ 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 server/helpers/custom-validators/video-imports.ts create mode 100644 server/helpers/youtube-dl.ts (limited to 'server/helpers') diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index d97bbd2a9..c6a350236 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) { } function sanitizeAndCheckVideoTorrentObject (video: any) { - if (video.type !== 'Video') return false + if (!video || video.type !== 'Video') return false if (!setValidRemoteTags(video)) return false if (!setValidRemoteVideoUrls(video)) return false diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts new file mode 100644 index 000000000..36c0559fd --- /dev/null +++ b/server/helpers/custom-validators/video-imports.ts @@ -0,0 +1,30 @@ +import 'express-validator' +import 'multer' +import * as validator from 'validator' +import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' +import { exists } from './misc' + +function isVideoImportTargetUrlValid (url: string) { + const isURLOptions = { + require_host: true, + require_tld: true, + require_protocol: true, + require_valid_protocol: true, + protocols: [ 'http', 'https' ] + } + + return exists(url) && + validator.isURL('' + url, isURLOptions) && + validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) +} + +function isVideoImportStateValid (value: any) { + return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined +} + +// --------------------------------------------------------------------------- + +export { + isVideoImportStateValid, + isVideoImportTargetUrlValid +} diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 04a19a9c6..480c5b49e 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) { } const consoleLoggerFormat = winston.format.printf(info => { - let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) + let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2) if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' else additionalInfos = ' ' + additionalInfos diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts new file mode 100644 index 000000000..74d3e213b --- /dev/null +++ b/server/helpers/youtube-dl.ts @@ -0,0 +1,142 @@ +import * as youtubeDL from 'youtube-dl' +import { truncate } from 'lodash' +import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' +import { join } from 'path' +import * as crypto from 'crypto' +import { logger } from './logger' + +export type YoutubeDLInfo = { + name: string + description: string + category: number + licence: number + nsfw: boolean + tags: string[] + thumbnailUrl: string +} + +function getYoutubeDLInfo (url: string): Promise { + return new Promise((res, rej) => { + const options = [ '-j', '--flat-playlist' ] + + youtubeDL.getInfo(url, options, (err, info) => { + if (err) return rej(err) + + const obj = normalizeObject(info) + + return res(buildVideoInfo(obj)) + }) + }) +} + +function downloadYoutubeDLVideo (url: string) { + const hash = crypto.createHash('sha256').update(url).digest('base64') + const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') + + logger.info('Importing video %s', url) + + const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] + + return new Promise((res, rej) => { + youtubeDL.exec(url, options, async (err, output) => { + if (err) return rej(err) + + return res(path) + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + downloadYoutubeDLVideo, + getYoutubeDLInfo +} + +// --------------------------------------------------------------------------- + +function normalizeObject (obj: any) { + const newObj: any = {} + + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue + + const value = obj[key] + + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } + + return newObj +} + +function buildVideoInfo (obj: any) { + return { + name: titleTruncation(obj.title), + description: descriptionTruncation(obj.description), + category: getCategory(obj.categories), + licence: getLicence(obj.license), + nsfw: isNSFW(obj), + tags: getTags(obj.tags), + thumbnailUrl: obj.thumbnail || undefined + } +} + +function titleTruncation (title: string) { + return truncate(title, { + 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max, + 'separator': /,? +/, + 'omission': ' […]' + }) +} + +function descriptionTruncation (description: string) { + if (!description) return undefined + + return truncate(description, { + 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, + 'separator': /,? +/, + 'omission': ' […]' + }) +} + +function isNSFW (info: any) { + return info.age_limit && info.age_limit >= 16 +} + +function getTags (tags: any) { + if (Array.isArray(tags) === false) return [] + + return tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) + .map(t => t.normalize()) + .slice(0, 5) +} + +function getLicence (licence: string) { + if (!licence) return undefined + + if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 + + return undefined +} + +function getCategory (categories: string[]) { + if (!categories) return undefined + + const categoryString = categories[0] + if (!categoryString || typeof categoryString !== 'string') return undefined + + if (categoryString === 'News & Politics') return 11 + + for (const key of Object.keys(VIDEO_CATEGORIES)) { + const category = VIDEO_CATEGORIES[key] + if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) + } + + return undefined +} -- cgit v1.2.3