From fbad87b0472f574409f7aa3ae7f8b54927d0cdd6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 2 Aug 2018 15:34:09 +0200 Subject: Add ability to import video with youtube-dl --- server/controllers/api/videos/import.ts | 151 +++++++++++++++++++++ server/controllers/api/videos/index.ts | 3 +- .../custom-validators/activitypub/videos.ts | 2 +- server/helpers/custom-validators/video-imports.ts | 30 ++++ server/helpers/logger.ts | 2 +- server/helpers/youtube-dl.ts | 142 +++++++++++++++++++ server/initializers/constants.ts | 18 ++- server/initializers/database.ts | 4 +- server/lib/job-queue/handlers/video-import.ts | 129 ++++++++++++++++++ server/lib/job-queue/job-queue.ts | 10 +- server/middlewares/validators/index.ts | 1 + server/middlewares/validators/video-imports.ts | 51 +++++++ server/middlewares/validators/videos.ts | 62 +++++---- server/models/account/account.ts | 1 - server/models/video/video-import.ts | 105 ++++++++++++++ server/models/video/video.ts | 2 +- 16 files changed, 672 insertions(+), 41 deletions(-) create mode 100644 server/controllers/api/videos/import.ts create mode 100644 server/helpers/custom-validators/video-imports.ts create mode 100644 server/helpers/youtube-dl.ts create mode 100644 server/lib/job-queue/handlers/video-import.ts create mode 100644 server/middlewares/validators/video-imports.ts create mode 100644 server/models/video/video-import.ts (limited to 'server') diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts new file mode 100644 index 000000000..9761cdbcf --- /dev/null +++ b/server/controllers/api/videos/import.ts @@ -0,0 +1,151 @@ +import * as express from 'express' +import { auditLoggerFactory } from '../../../helpers/audit-logger' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + videoImportAddValidator, + videoImportDeleteValidator +} from '../../../middlewares' +import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers' +import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' +import { createReqFiles } from '../../../helpers/express-utils' +import { logger } from '../../../helpers/logger' +import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' +import { VideoModel } from '../../../models/video/video' +import { getVideoActivityPubUrl } from '../../../lib/activitypub' +import { TagModel } from '../../../models/video/tag' +import { VideoImportModel } from '../../../models/video/video-import' +import { JobQueue } from '../../../lib/job-queue/job-queue' +import { processImage } from '../../../helpers/image-utils' +import { join } from 'path' + +const auditLogger = auditLoggerFactory('video-imports') +const videoImportsRouter = express.Router() + +const reqVideoFileImport = createReqFiles( + [ 'thumbnailfile', 'previewfile' ], + IMAGE_MIMETYPE_EXT, + { + thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, + previewfile: CONFIG.STORAGE.PREVIEWS_DIR + } +) + +videoImportsRouter.post('/imports', + authenticate, + reqVideoFileImport, + asyncMiddleware(videoImportAddValidator), + asyncRetryTransactionMiddleware(addVideoImport) +) + +videoImportsRouter.delete('/imports/:id', + authenticate, + videoImportDeleteValidator, + asyncRetryTransactionMiddleware(deleteVideoImport) +) + +// --------------------------------------------------------------------------- + +export { + videoImportsRouter +} + +// --------------------------------------------------------------------------- + +async function addVideoImport (req: express.Request, res: express.Response) { + const body: VideoImportCreate = req.body + const targetUrl = body.targetUrl + + let youtubeDLInfo: YoutubeDLInfo + try { + youtubeDLInfo = await getYoutubeDLInfo(targetUrl) + } catch (err) { + logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) + + return res.status(400).json({ + error: 'Cannot fetch remote information of this URL.' + }).end() + } + + // Create video DB object + const videoData = { + name: body.name || youtubeDLInfo.name, + remote: false, + category: body.category || youtubeDLInfo.category, + licence: body.licence || youtubeDLInfo.licence, + language: undefined, + commentsEnabled: body.commentsEnabled || true, + waitTranscoding: body.waitTranscoding || false, + state: VideoState.TO_IMPORT, + nsfw: body.nsfw || youtubeDLInfo.nsfw || false, + description: body.description || youtubeDLInfo.description, + support: body.support || null, + privacy: body.privacy || VideoPrivacy.PRIVATE, + duration: 0, // duration will be set by the import job + channelId: res.locals.videoChannel.id + } + const video = new VideoModel(videoData) + video.url = getVideoActivityPubUrl(video) + + // Process thumbnail file? + const thumbnailField = req.files['thumbnailfile'] + let downloadThumbnail = true + if (thumbnailField) { + const thumbnailPhysicalFile = thumbnailField[ 0 ] + await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) + downloadThumbnail = false + } + + // Process preview file? + const previewField = req.files['previewfile'] + let downloadPreview = true + if (previewField) { + const previewPhysicalFile = previewField[0] + await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) + downloadPreview = false + } + + const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + // Save video object in database + const videoCreated = await video.save(sequelizeOptions) + videoCreated.VideoChannel = res.locals.videoChannel + + // Set tags to the video + if (youtubeDLInfo.tags !== undefined) { + const tagInstances = await TagModel.findOrCreateTags(youtubeDLInfo.tags, t) + + await videoCreated.$set('Tags', tagInstances, sequelizeOptions) + videoCreated.Tags = tagInstances + } + + // Create video import object in database + const videoImport = await VideoImportModel.create({ + targetUrl, + state: VideoImportState.PENDING, + videoId: videoCreated.id + }, sequelizeOptions) + + videoImport.Video = videoCreated + + return videoImport + }) + + // Create job to import the video + const payload = { + type: 'youtube-dl' as 'youtube-dl', + videoImportId: videoImport.id, + thumbnailUrl: youtubeDLInfo.thumbnailUrl, + downloadThumbnail, + downloadPreview + } + await JobQueue.Instance.createJob({ type: 'video-import', payload }) + + return res.json(videoImport.toFormattedJSON()) +} + +async function deleteVideoImport (req: express.Request, res: express.Response) { + // TODO: delete video import +} diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index e396ee6be..c9365da08 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -54,6 +54,7 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type' import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils' import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' import { videoCaptionsRouter } from './captions' +import { videoImportsRouter } from './import' const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() @@ -81,6 +82,7 @@ videosRouter.use('/', blacklistRouter) videosRouter.use('/', rateVideoRouter) videosRouter.use('/', videoCommentRouter) videosRouter.use('/', videoCaptionsRouter) +videosRouter.use('/', videoImportsRouter) videosRouter.get('/categories', listVideoCategories) videosRouter.get('/licences', listVideoLicences) @@ -160,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) { const videoData = { name: videoInfo.name, remote: false, - extname: extname(videoPhysicalFile.filename), category: videoInfo.category, licence: videoInfo.licence, language: videoInfo.language, diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index d97bbd2a9..c6a350236 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) { } function sanitizeAndCheckVideoTorrentObject (video: any) { - if (video.type !== 'Video') return false + if (!video || video.type !== 'Video') return false if (!setValidRemoteTags(video)) return false if (!setValidRemoteVideoUrls(video)) return false diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts new file mode 100644 index 000000000..36c0559fd --- /dev/null +++ b/server/helpers/custom-validators/video-imports.ts @@ -0,0 +1,30 @@ +import 'express-validator' +import 'multer' +import * as validator from 'validator' +import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' +import { exists } from './misc' + +function isVideoImportTargetUrlValid (url: string) { + const isURLOptions = { + require_host: true, + require_tld: true, + require_protocol: true, + require_valid_protocol: true, + protocols: [ 'http', 'https' ] + } + + return exists(url) && + validator.isURL('' + url, isURLOptions) && + validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) +} + +function isVideoImportStateValid (value: any) { + return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined +} + +// --------------------------------------------------------------------------- + +export { + isVideoImportStateValid, + isVideoImportTargetUrlValid +} diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 04a19a9c6..480c5b49e 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) { } const consoleLoggerFormat = winston.format.printf(info => { - let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) + let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2) if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' else additionalInfos = ' ' + additionalInfos diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts new file mode 100644 index 000000000..74d3e213b --- /dev/null +++ b/server/helpers/youtube-dl.ts @@ -0,0 +1,142 @@ +import * as youtubeDL from 'youtube-dl' +import { truncate } from 'lodash' +import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' +import { join } from 'path' +import * as crypto from 'crypto' +import { logger } from './logger' + +export type YoutubeDLInfo = { + name: string + description: string + category: number + licence: number + nsfw: boolean + tags: string[] + thumbnailUrl: string +} + +function getYoutubeDLInfo (url: string): Promise { + return new Promise((res, rej) => { + const options = [ '-j', '--flat-playlist' ] + + youtubeDL.getInfo(url, options, (err, info) => { + if (err) return rej(err) + + const obj = normalizeObject(info) + + return res(buildVideoInfo(obj)) + }) + }) +} + +function downloadYoutubeDLVideo (url: string) { + const hash = crypto.createHash('sha256').update(url).digest('base64') + const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') + + logger.info('Importing video %s', url) + + const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] + + return new Promise((res, rej) => { + youtubeDL.exec(url, options, async (err, output) => { + if (err) return rej(err) + + return res(path) + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + downloadYoutubeDLVideo, + getYoutubeDLInfo +} + +// --------------------------------------------------------------------------- + +function normalizeObject (obj: any) { + const newObj: any = {} + + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue + + const value = obj[key] + + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } + + return newObj +} + +function buildVideoInfo (obj: any) { + return { + name: titleTruncation(obj.title), + description: descriptionTruncation(obj.description), + category: getCategory(obj.categories), + licence: getLicence(obj.license), + nsfw: isNSFW(obj), + tags: getTags(obj.tags), + thumbnailUrl: obj.thumbnail || undefined + } +} + +function titleTruncation (title: string) { + return truncate(title, { + 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max, + 'separator': /,? +/, + 'omission': ' […]' + }) +} + +function descriptionTruncation (description: string) { + if (!description) return undefined + + return truncate(description, { + 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, + 'separator': /,? +/, + 'omission': ' […]' + }) +} + +function isNSFW (info: any) { + return info.age_limit && info.age_limit >= 16 +} + +function getTags (tags: any) { + if (Array.isArray(tags) === false) return [] + + return tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) + .map(t => t.normalize()) + .slice(0, 5) +} + +function getLicence (licence: string) { + if (!licence) return undefined + + if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 + + return undefined +} + +function getCategory (categories: string[]) { + if (!categories) return undefined + + const categoryString = categories[0] + if (!categoryString || typeof categoryString !== 'string') return undefined + + if (categoryString === 'News & Politics') return 11 + + for (const key of Object.keys(VIDEO_CATEGORIES)) { + const category = VIDEO_CATEGORIES[key] + if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) + } + + return undefined +} diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index bec343bb7..fdd772d84 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos' import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { invert } from 'lodash' +import { VideoImportState } from '../../shared/models/videos/video-import-state.enum' // Use a variable to reload the configuration if we need let config: IConfig = require('config') @@ -85,6 +86,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = { 'activitypub-follow': 5, 'video-file-import': 1, 'video-file': 1, + 'video-import': 1, 'email': 5 } const JOB_CONCURRENCY: { [ id in JobType ]: number } = { @@ -94,6 +96,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = { 'activitypub-follow': 3, 'video-file-import': 1, 'video-file': 1, + 'video-import': 1, 'email': 5 } const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job @@ -248,6 +251,9 @@ const CONSTRAINTS_FIELDS = { } } }, + VIDEO_IMPORTS: { + URL: { min: 3, max: 2000 } // Length + }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length LANGUAGE: { min: 1, max: 10 }, // Length @@ -262,7 +268,7 @@ const CONSTRAINTS_FIELDS = { }, EXTNAME: [ '.mp4', '.ogv', '.webm' ], INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 - DURATION: { min: 1 }, // Number + DURATION: { min: 0 }, // Number TAGS: { min: 0, max: 5 }, // Number of total tags TAG: { min: 2, max: 30 }, // Length THUMBNAIL: { min: 2, max: 30 }, @@ -363,7 +369,14 @@ const VIDEO_PRIVACIES = { const VIDEO_STATES = { [VideoState.PUBLISHED]: 'Published', - [VideoState.TO_TRANSCODE]: 'To transcode' + [VideoState.TO_TRANSCODE]: 'To transcode', + [VideoState.TO_IMPORT]: 'To import' +} + +const VIDEO_IMPORT_STATES = { + [VideoImportState.FAILED]: 'Failed', + [VideoImportState.PENDING]: 'Pending', + [VideoImportState.SUCCESS]: 'Success' } const VIDEO_MIMETYPE_EXT = { @@ -585,6 +598,7 @@ export { RATES_LIMIT, VIDEO_EXT_MIMETYPE, JOB_COMPLETED_LIFETIME, + VIDEO_IMPORT_STATES, VIDEO_VIEW_LIFETIME, buildLanguages } diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 21c083084..0be752363 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag' import { CONFIG } from './constants' import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' import { VideoCaptionModel } from '../models/video/video-caption' +import { VideoImportModel } from '../models/video/video-import' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -81,7 +82,8 @@ async function initDatabaseModels (silent: boolean) { VideoTagModel, VideoModel, VideoCommentModel, - ScheduleVideoUpdateModel + ScheduleVideoUpdateModel, + VideoImportModel ]) // Check extensions exist in the database diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts new file mode 100644 index 000000000..2f219e986 --- /dev/null +++ b/server/lib/job-queue/handlers/video-import.ts @@ -0,0 +1,129 @@ +import * as Bull from 'bull' +import { logger } from '../../../helpers/logger' +import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' +import { VideoImportModel } from '../../../models/video/video-import' +import { VideoImportState } from '../../../../shared/models/videos' +import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' +import { extname, join } from 'path' +import { VideoFileModel } from '../../../models/video/video-file' +import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils' +import { CONFIG, sequelizeTypescript } from '../../../initializers' +import { doRequestAndSaveToFile } from '../../../helpers/requests' +import { VideoState } from '../../../../shared' +import { JobQueue } from '../index' +import { federateVideoIfNeeded } from '../../activitypub' + +export type VideoImportPayload = { + type: 'youtube-dl' + videoImportId: number + thumbnailUrl: string + downloadThumbnail: boolean + downloadPreview: boolean +} + +async function processVideoImport (job: Bull.Job) { + const payload = job.data as VideoImportPayload + logger.info('Processing video import in job %d.', job.id) + + const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) + if (!videoImport) throw new Error('Cannot import video %s: the video import entry does not exist anymore.') + + let tempVideoPath: string + try { + // Download video from youtubeDL + tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) + + // Get information about this video + const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) + const fps = await getVideoFileFPS(tempVideoPath) + const stats = await statPromise(tempVideoPath) + const duration = await getDurationFromVideoFile(tempVideoPath) + + // Create video file object in database + const videoFileData = { + extname: extname(tempVideoPath), + resolution: videoFileResolution, + size: stats.size, + fps, + videoId: videoImport.videoId + } + const videoFile = new VideoFileModel(videoFileData) + + // Move file + const destination = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) + await renamePromise(tempVideoPath, destination) + + // Process thumbnail + if (payload.downloadThumbnail) { + if (payload.thumbnailUrl) { + const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) + await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) + } else { + await videoImport.Video.createThumbnail(videoFile) + } + } + + // Process preview + if (payload.downloadPreview) { + if (payload.thumbnailUrl) { + const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) + await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) + } else { + await videoImport.Video.createPreview(videoFile) + } + } + + // Create torrent + await videoImport.Video.createTorrentAndSetInfoHash(videoFile) + + const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => { + await videoFile.save({ transaction: t }) + + // Update video DB object + videoImport.Video.duration = duration + videoImport.Video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED + const videoUpdated = await videoImport.Video.save({ transaction: t }) + + // Now we can federate the video + await federateVideoIfNeeded(videoImport.Video, true, t) + + // Update video import object + videoImport.state = VideoImportState.SUCCESS + const videoImportUpdated = await videoImport.save({ transaction: t }) + + logger.info('Video %s imported.', videoImport.targetUrl) + + videoImportUpdated.Video = videoUpdated + return videoImportUpdated + }) + + // Create transcoding jobs? + if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { + // Put uuid because we don't have id auto incremented for now + const dataInput = { + videoUUID: videoImportUpdated.Video.uuid, + isNewVideo: true + } + + await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) + } + + } catch (err) { + try { + if (tempVideoPath) await unlinkPromise(tempVideoPath) + } catch (errUnlink) { + logger.error('Cannot cleanup files after a video import error.', { err: errUnlink }) + } + + videoImport.state = VideoImportState.FAILED + await videoImport.save() + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + processVideoImport +} diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 8ff0c169e..2e14867f2 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts @@ -9,6 +9,7 @@ import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './ import { EmailPayload, processEmail } from './handlers/email' import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' +import { processVideoImport, VideoImportPayload } from './handlers/video-import' type CreateJobArgument = { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | @@ -17,7 +18,8 @@ type CreateJobArgument = { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | { type: 'video-file-import', payload: VideoFileImportPayload } | { type: 'video-file', payload: VideoFilePayload } | - { type: 'email', payload: EmailPayload } + { type: 'email', payload: EmailPayload } | + { type: 'video-import', payload: VideoImportPayload } const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-http-broadcast': processActivityPubHttpBroadcast, @@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise} = { 'activitypub-follow': processActivityPubFollow, 'video-file-import': processVideoFileImport, 'video-file': processVideoFile, - 'email': processEmail + 'email': processEmail, + 'video-import': processVideoImport } const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { @@ -43,7 +46,8 @@ const jobTypes: JobType[] = [ 'activitypub-http-unicast', 'email', 'video-file', - 'video-file-import' + 'video-file-import', + 'video-import' ] class JobQueue { diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index e3f0f5963..c5400c8f5 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -11,3 +11,4 @@ export * from './video-blacklist' export * from './video-channels' export * from './webfinger' export * from './search' +export * from './video-imports' diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts new file mode 100644 index 000000000..0ba759ff0 --- /dev/null +++ b/server/middlewares/validators/video-imports.ts @@ -0,0 +1,51 @@ +import * as express from 'express' +import { body, param } from 'express-validator/check' +import { isIdValid } from '../../helpers/custom-validators/misc' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' +import { getCommonVideoAttributes } from './videos' +import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' +import { cleanUpReqFiles } from '../../helpers/utils' +import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' + +const videoImportAddValidator = getCommonVideoAttributes().concat([ + body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), + body('channelId') + .toInt() + .custom(isIdValid).withMessage('Should have correct video channel id'), + body('name') + .optional() + .custom(isVideoNameValid).withMessage('Should have a valid name'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body }) + + const user = res.locals.oauth.token.User + + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +const videoImportDeleteValidator = [ + param('id').custom(isIdValid).not().isEmpty().withMessage('Should have a valid id'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoImportDeleteValidator parameters', { parameters: req.body }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoImportAddValidator, + videoImportDeleteValidator +} + +// --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index 9357c1e39..c812d4677 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts @@ -223,36 +223,6 @@ const videosShareValidator = [ } ] -// --------------------------------------------------------------------------- - -export { - videosAddValidator, - videosUpdateValidator, - videosGetValidator, - videosRemoveValidator, - videosShareValidator, - - videoAbuseReportValidator, - - videoRateValidator -} - -// --------------------------------------------------------------------------- - -function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { - if (req.body.scheduleUpdate) { - if (!req.body.scheduleUpdate.updateAt) { - res.status(400) - .json({ error: 'Schedule update at is mandatory.' }) - .end() - - return true - } - } - - return false -} - function getCommonVideoAttributes () { return [ body('thumbnailfile') @@ -319,3 +289,35 @@ function getCommonVideoAttributes () { .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy') ] as (ValidationChain | express.Handler)[] } + +// --------------------------------------------------------------------------- + +export { + videosAddValidator, + videosUpdateValidator, + videosGetValidator, + videosRemoveValidator, + videosShareValidator, + + videoAbuseReportValidator, + + videoRateValidator, + + getCommonVideoAttributes +} + +// --------------------------------------------------------------------------- + +function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { + if (req.body.scheduleUpdate) { + if (!req.body.scheduleUpdate.updateAt) { + res.status(400) + .json({ error: 'Schedule update at is mandatory.' }) + .end() + + return true + } + } + + return false +} diff --git a/server/models/account/account.ts b/server/models/account/account.ts index d674d8d22..66f5dcf2e 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -16,7 +16,6 @@ import { } from 'sequelize-typescript' import { Account } from '../../../shared/models/actors' import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' -import { logger } from '../../helpers/logger' import { sendDeleteActor } from '../../lib/activitypub/send' import { ActorModel } from '../activitypub/actor' import { ApplicationModel } from '../application/application' diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts new file mode 100644 index 000000000..89eeafd6a --- /dev/null +++ b/server/models/video/video-import.ts @@ -0,0 +1,105 @@ +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { CONSTRAINTS_FIELDS } from '../../initializers' +import { throwIfNotValid } from '../utils' +import { VideoModel } from './video' +import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' +import { VideoImport, VideoImportState } from '../../../shared' +import { VideoChannelModel } from './video-channel' +import { AccountModel } from '../account/account' + +@DefaultScope({ + include: [ + { + model: () => VideoModel, + required: true, + include: [ + { + model: () => VideoChannelModel, + required: true, + include: [ + { + model: () => AccountModel, + required: true + } + ] + } + ] + } + ] +}) + +@Table({ + tableName: 'videoImport', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + } + ] +}) +export class VideoImportModel extends Model { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) + targetUrl: string + + @AllowNull(false) + @Default(null) + @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) + @Column + state: VideoImportState + + @AllowNull(true) + @Default(null) + @Column(DataType.TEXT) + error: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: VideoModel + + static loadAndPopulateVideo (id: number) { + return VideoImportModel.findById(id) + } + + toFormattedJSON (): VideoImport { + const videoFormatOptions = { + additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true } + } + const video = Object.assign(this.Video.toFormattedJSON(videoFormatOptions), { + tags: this.Video.Tags.map(t => t.name) + }) + + return { + targetUrl: this.targetUrl, + video + } + } +} diff --git a/server/models/video/video.ts b/server/models/video/video.ts index a6c4620b2..459fcb31e 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -377,7 +377,7 @@ type AvailableForListOptions = { include: [ { model: () => VideoFileModel.unscoped(), - required: true + required: false } ] }, -- cgit v1.2.3