aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/import.ts151
-rw-r--r--server/controllers/api/videos/index.ts3
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts2
-rw-r--r--server/helpers/custom-validators/video-imports.ts30
-rw-r--r--server/helpers/logger.ts2
-rw-r--r--server/helpers/youtube-dl.ts142
-rw-r--r--server/initializers/constants.ts18
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/lib/job-queue/handlers/video-import.ts129
-rw-r--r--server/lib/job-queue/job-queue.ts10
-rw-r--r--server/middlewares/validators/index.ts1
-rw-r--r--server/middlewares/validators/video-imports.ts51
-rw-r--r--server/middlewares/validators/videos.ts62
-rw-r--r--server/models/account/account.ts1
-rw-r--r--server/models/video/video-import.ts105
-rw-r--r--server/models/video/video.ts2
16 files changed, 672 insertions, 41 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
new file mode 100644
index 000000000..9761cdbcf
--- /dev/null
+++ b/server/controllers/api/videos/import.ts
@@ -0,0 +1,151 @@
1import * as express from 'express'
2import { auditLoggerFactory } from '../../../helpers/audit-logger'
3import {
4 asyncMiddleware,
5 asyncRetryTransactionMiddleware,
6 authenticate,
7 videoImportAddValidator,
8 videoImportDeleteValidator
9} from '../../../middlewares'
10import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
11import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
12import { createReqFiles } from '../../../helpers/express-utils'
13import { logger } from '../../../helpers/logger'
14import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
15import { VideoModel } from '../../../models/video/video'
16import { getVideoActivityPubUrl } from '../../../lib/activitypub'
17import { TagModel } from '../../../models/video/tag'
18import { VideoImportModel } from '../../../models/video/video-import'
19import { JobQueue } from '../../../lib/job-queue/job-queue'
20import { processImage } from '../../../helpers/image-utils'
21import { join } from 'path'
22
23const auditLogger = auditLoggerFactory('video-imports')
24const videoImportsRouter = express.Router()
25
26const reqVideoFileImport = createReqFiles(
27 [ 'thumbnailfile', 'previewfile' ],
28 IMAGE_MIMETYPE_EXT,
29 {
30 thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
31 previewfile: CONFIG.STORAGE.PREVIEWS_DIR
32 }
33)
34
35videoImportsRouter.post('/imports',
36 authenticate,
37 reqVideoFileImport,
38 asyncMiddleware(videoImportAddValidator),
39 asyncRetryTransactionMiddleware(addVideoImport)
40)
41
42videoImportsRouter.delete('/imports/:id',
43 authenticate,
44 videoImportDeleteValidator,
45 asyncRetryTransactionMiddleware(deleteVideoImport)
46)
47
48// ---------------------------------------------------------------------------
49
50export {
51 videoImportsRouter
52}
53
54// ---------------------------------------------------------------------------
55
56async function addVideoImport (req: express.Request, res: express.Response) {
57 const body: VideoImportCreate = req.body
58 const targetUrl = body.targetUrl
59
60 let youtubeDLInfo: YoutubeDLInfo
61 try {
62 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
63 } catch (err) {
64 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
65
66 return res.status(400).json({
67 error: 'Cannot fetch remote information of this URL.'
68 }).end()
69 }
70
71 // Create video DB object
72 const videoData = {
73 name: body.name || youtubeDLInfo.name,
74 remote: false,
75 category: body.category || youtubeDLInfo.category,
76 licence: body.licence || youtubeDLInfo.licence,
77 language: undefined,
78 commentsEnabled: body.commentsEnabled || true,
79 waitTranscoding: body.waitTranscoding || false,
80 state: VideoState.TO_IMPORT,
81 nsfw: body.nsfw || youtubeDLInfo.nsfw || false,
82 description: body.description || youtubeDLInfo.description,
83 support: body.support || null,
84 privacy: body.privacy || VideoPrivacy.PRIVATE,
85 duration: 0, // duration will be set by the import job
86 channelId: res.locals.videoChannel.id
87 }
88 const video = new VideoModel(videoData)
89 video.url = getVideoActivityPubUrl(video)
90
91 // Process thumbnail file?
92 const thumbnailField = req.files['thumbnailfile']
93 let downloadThumbnail = true
94 if (thumbnailField) {
95 const thumbnailPhysicalFile = thumbnailField[ 0 ]
96 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
97 downloadThumbnail = false
98 }
99
100 // Process preview file?
101 const previewField = req.files['previewfile']
102 let downloadPreview = true
103 if (previewField) {
104 const previewPhysicalFile = previewField[0]
105 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
106 downloadPreview = false
107 }
108
109 const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => {
110 const sequelizeOptions = { transaction: t }
111
112 // Save video object in database
113 const videoCreated = await video.save(sequelizeOptions)
114 videoCreated.VideoChannel = res.locals.videoChannel
115
116 // Set tags to the video
117 if (youtubeDLInfo.tags !== undefined) {
118 const tagInstances = await TagModel.findOrCreateTags(youtubeDLInfo.tags, t)
119
120 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
121 videoCreated.Tags = tagInstances
122 }
123
124 // Create video import object in database
125 const videoImport = await VideoImportModel.create({
126 targetUrl,
127 state: VideoImportState.PENDING,
128 videoId: videoCreated.id
129 }, sequelizeOptions)
130
131 videoImport.Video = videoCreated
132
133 return videoImport
134 })
135
136 // Create job to import the video
137 const payload = {
138 type: 'youtube-dl' as 'youtube-dl',
139 videoImportId: videoImport.id,
140 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
141 downloadThumbnail,
142 downloadPreview
143 }
144 await JobQueue.Instance.createJob({ type: 'video-import', payload })
145
146 return res.json(videoImport.toFormattedJSON())
147}
148
149async function deleteVideoImport (req: express.Request, res: express.Response) {
150 // TODO: delete video import
151}
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index e396ee6be..c9365da08 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -54,6 +54,7 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
54import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils' 54import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils'
55import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' 55import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
56import { videoCaptionsRouter } from './captions' 56import { videoCaptionsRouter } from './captions'
57import { videoImportsRouter } from './import'
57 58
58const auditLogger = auditLoggerFactory('videos') 59const auditLogger = auditLoggerFactory('videos')
59const videosRouter = express.Router() 60const videosRouter = express.Router()
@@ -81,6 +82,7 @@ videosRouter.use('/', blacklistRouter)
81videosRouter.use('/', rateVideoRouter) 82videosRouter.use('/', rateVideoRouter)
82videosRouter.use('/', videoCommentRouter) 83videosRouter.use('/', videoCommentRouter)
83videosRouter.use('/', videoCaptionsRouter) 84videosRouter.use('/', videoCaptionsRouter)
85videosRouter.use('/', videoImportsRouter)
84 86
85videosRouter.get('/categories', listVideoCategories) 87videosRouter.get('/categories', listVideoCategories)
86videosRouter.get('/licences', listVideoLicences) 88videosRouter.get('/licences', listVideoLicences)
@@ -160,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) {
160 const videoData = { 162 const videoData = {
161 name: videoInfo.name, 163 name: videoInfo.name,
162 remote: false, 164 remote: false,
163 extname: extname(videoPhysicalFile.filename),
164 category: videoInfo.category, 165 category: videoInfo.category,
165 licence: videoInfo.licence, 166 licence: videoInfo.licence,
166 language: videoInfo.language, 167 language: videoInfo.language,
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index d97bbd2a9..c6a350236 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) {
45} 45}
46 46
47function sanitizeAndCheckVideoTorrentObject (video: any) { 47function sanitizeAndCheckVideoTorrentObject (video: any) {
48 if (video.type !== 'Video') return false 48 if (!video || video.type !== 'Video') return false
49 49
50 if (!setValidRemoteTags(video)) return false 50 if (!setValidRemoteTags(video)) return false
51 if (!setValidRemoteVideoUrls(video)) return false 51 if (!setValidRemoteVideoUrls(video)) return false
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
new file mode 100644
index 000000000..36c0559fd
--- /dev/null
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -0,0 +1,30 @@
1import 'express-validator'
2import 'multer'
3import * as validator from 'validator'
4import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
5import { exists } from './misc'
6
7function isVideoImportTargetUrlValid (url: string) {
8 const isURLOptions = {
9 require_host: true,
10 require_tld: true,
11 require_protocol: true,
12 require_valid_protocol: true,
13 protocols: [ 'http', 'https' ]
14 }
15
16 return exists(url) &&
17 validator.isURL('' + url, isURLOptions) &&
18 validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
19}
20
21function isVideoImportStateValid (value: any) {
22 return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined
23}
24
25// ---------------------------------------------------------------------------
26
27export {
28 isVideoImportStateValid,
29 isVideoImportTargetUrlValid
30}
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 04a19a9c6..480c5b49e 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) {
22} 22}
23 23
24const consoleLoggerFormat = winston.format.printf(info => { 24const consoleLoggerFormat = winston.format.printf(info => {
25 let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) 25 let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2)
26 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' 26 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
27 else additionalInfos = ' ' + additionalInfos 27 else additionalInfos = ' ' + additionalInfos
28 28
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
new file mode 100644
index 000000000..74d3e213b
--- /dev/null
+++ b/server/helpers/youtube-dl.ts
@@ -0,0 +1,142 @@
1import * as youtubeDL from 'youtube-dl'
2import { truncate } from 'lodash'
3import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
4import { join } from 'path'
5import * as crypto from 'crypto'
6import { logger } from './logger'
7
8export type YoutubeDLInfo = {
9 name: string
10 description: string
11 category: number
12 licence: number
13 nsfw: boolean
14 tags: string[]
15 thumbnailUrl: string
16}
17
18function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
19 return new Promise<YoutubeDLInfo>((res, rej) => {
20 const options = [ '-j', '--flat-playlist' ]
21
22 youtubeDL.getInfo(url, options, (err, info) => {
23 if (err) return rej(err)
24
25 const obj = normalizeObject(info)
26
27 return res(buildVideoInfo(obj))
28 })
29 })
30}
31
32function downloadYoutubeDLVideo (url: string) {
33 const hash = crypto.createHash('sha256').update(url).digest('base64')
34 const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
35
36 logger.info('Importing video %s', url)
37
38 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
39
40 return new Promise<string>((res, rej) => {
41 youtubeDL.exec(url, options, async (err, output) => {
42 if (err) return rej(err)
43
44 return res(path)
45 })
46 })
47}
48
49// ---------------------------------------------------------------------------
50
51export {
52 downloadYoutubeDLVideo,
53 getYoutubeDLInfo
54}
55
56// ---------------------------------------------------------------------------
57
58function normalizeObject (obj: any) {
59 const newObj: any = {}
60
61 for (const key of Object.keys(obj)) {
62 // Deprecated key
63 if (key === 'resolution') continue
64
65 const value = obj[key]
66
67 if (typeof value === 'string') {
68 newObj[key] = value.normalize()
69 } else {
70 newObj[key] = value
71 }
72 }
73
74 return newObj
75}
76
77function buildVideoInfo (obj: any) {
78 return {
79 name: titleTruncation(obj.title),
80 description: descriptionTruncation(obj.description),
81 category: getCategory(obj.categories),
82 licence: getLicence(obj.license),
83 nsfw: isNSFW(obj),
84 tags: getTags(obj.tags),
85 thumbnailUrl: obj.thumbnail || undefined
86 }
87}
88
89function titleTruncation (title: string) {
90 return truncate(title, {
91 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
92 'separator': /,? +/,
93 'omission': ' […]'
94 })
95}
96
97function descriptionTruncation (description: string) {
98 if (!description) return undefined
99
100 return truncate(description, {
101 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
102 'separator': /,? +/,
103 'omission': ' […]'
104 })
105}
106
107function isNSFW (info: any) {
108 return info.age_limit && info.age_limit >= 16
109}
110
111function getTags (tags: any) {
112 if (Array.isArray(tags) === false) return []
113
114 return tags
115 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
116 .map(t => t.normalize())
117 .slice(0, 5)
118}
119
120function getLicence (licence: string) {
121 if (!licence) return undefined
122
123 if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
124
125 return undefined
126}
127
128function getCategory (categories: string[]) {
129 if (!categories) return undefined
130
131 const categoryString = categories[0]
132 if (!categoryString || typeof categoryString !== 'string') return undefined
133
134 if (categoryString === 'News & Politics') return 11
135
136 for (const key of Object.keys(VIDEO_CATEGORIES)) {
137 const category = VIDEO_CATEGORIES[key]
138 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
139 }
140
141 return undefined
142}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index bec343bb7..fdd772d84 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos'
8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' 8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
10import { invert } from 'lodash' 10import { invert } from 'lodash'
11import { VideoImportState } from '../../shared/models/videos/video-import-state.enum'
11 12
12// Use a variable to reload the configuration if we need 13// Use a variable to reload the configuration if we need
13let config: IConfig = require('config') 14let config: IConfig = require('config')
@@ -85,6 +86,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
85 'activitypub-follow': 5, 86 'activitypub-follow': 5,
86 'video-file-import': 1, 87 'video-file-import': 1,
87 'video-file': 1, 88 'video-file': 1,
89 'video-import': 1,
88 'email': 5 90 'email': 5
89} 91}
90const JOB_CONCURRENCY: { [ id in JobType ]: number } = { 92const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
@@ -94,6 +96,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
94 'activitypub-follow': 3, 96 'activitypub-follow': 3,
95 'video-file-import': 1, 97 'video-file-import': 1,
96 'video-file': 1, 98 'video-file': 1,
99 'video-import': 1,
97 'email': 5 100 'email': 5
98} 101}
99const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job 102const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
@@ -248,6 +251,9 @@ const CONSTRAINTS_FIELDS = {
248 } 251 }
249 } 252 }
250 }, 253 },
254 VIDEO_IMPORTS: {
255 URL: { min: 3, max: 2000 } // Length
256 },
251 VIDEOS: { 257 VIDEOS: {
252 NAME: { min: 3, max: 120 }, // Length 258 NAME: { min: 3, max: 120 }, // Length
253 LANGUAGE: { min: 1, max: 10 }, // Length 259 LANGUAGE: { min: 1, max: 10 }, // Length
@@ -262,7 +268,7 @@ const CONSTRAINTS_FIELDS = {
262 }, 268 },
263 EXTNAME: [ '.mp4', '.ogv', '.webm' ], 269 EXTNAME: [ '.mp4', '.ogv', '.webm' ],
264 INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 270 INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
265 DURATION: { min: 1 }, // Number 271 DURATION: { min: 0 }, // Number
266 TAGS: { min: 0, max: 5 }, // Number of total tags 272 TAGS: { min: 0, max: 5 }, // Number of total tags
267 TAG: { min: 2, max: 30 }, // Length 273 TAG: { min: 2, max: 30 }, // Length
268 THUMBNAIL: { min: 2, max: 30 }, 274 THUMBNAIL: { min: 2, max: 30 },
@@ -363,7 +369,14 @@ const VIDEO_PRIVACIES = {
363 369
364const VIDEO_STATES = { 370const VIDEO_STATES = {
365 [VideoState.PUBLISHED]: 'Published', 371 [VideoState.PUBLISHED]: 'Published',
366 [VideoState.TO_TRANSCODE]: 'To transcode' 372 [VideoState.TO_TRANSCODE]: 'To transcode',
373 [VideoState.TO_IMPORT]: 'To import'
374}
375
376const VIDEO_IMPORT_STATES = {
377 [VideoImportState.FAILED]: 'Failed',
378 [VideoImportState.PENDING]: 'Pending',
379 [VideoImportState.SUCCESS]: 'Success'
367} 380}
368 381
369const VIDEO_MIMETYPE_EXT = { 382const VIDEO_MIMETYPE_EXT = {
@@ -585,6 +598,7 @@ export {
585 RATES_LIMIT, 598 RATES_LIMIT,
586 VIDEO_EXT_MIMETYPE, 599 VIDEO_EXT_MIMETYPE,
587 JOB_COMPLETED_LIFETIME, 600 JOB_COMPLETED_LIFETIME,
601 VIDEO_IMPORT_STATES,
588 VIDEO_VIEW_LIFETIME, 602 VIDEO_VIEW_LIFETIME,
589 buildLanguages 603 buildLanguages
590} 604}
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 21c083084..0be752363 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag'
24import { CONFIG } from './constants' 24import { CONFIG } from './constants'
25import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' 25import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update'
26import { VideoCaptionModel } from '../models/video/video-caption' 26import { VideoCaptionModel } from '../models/video/video-caption'
27import { VideoImportModel } from '../models/video/video-import'
27 28
28require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 29require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
29 30
@@ -81,7 +82,8 @@ async function initDatabaseModels (silent: boolean) {
81 VideoTagModel, 82 VideoTagModel,
82 VideoModel, 83 VideoModel,
83 VideoCommentModel, 84 VideoCommentModel,
84 ScheduleVideoUpdateModel 85 ScheduleVideoUpdateModel,
86 VideoImportModel
85 ]) 87 ])
86 88
87 // Check extensions exist in the database 89 // Check extensions exist in the database
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
new file mode 100644
index 000000000..2f219e986
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -0,0 +1,129 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
4import { VideoImportModel } from '../../../models/video/video-import'
5import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file'
9import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils'
10import { CONFIG, sequelizeTypescript } from '../../../initializers'
11import { doRequestAndSaveToFile } from '../../../helpers/requests'
12import { VideoState } from '../../../../shared'
13import { JobQueue } from '../index'
14import { federateVideoIfNeeded } from '../../activitypub'
15
16export type VideoImportPayload = {
17 type: 'youtube-dl'
18 videoImportId: number
19 thumbnailUrl: string
20 downloadThumbnail: boolean
21 downloadPreview: boolean
22}
23
24async function processVideoImport (job: Bull.Job) {
25 const payload = job.data as VideoImportPayload
26 logger.info('Processing video import in job %d.', job.id)
27
28 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
29 if (!videoImport) throw new Error('Cannot import video %s: the video import entry does not exist anymore.')
30
31 let tempVideoPath: string
32 try {
33 // Download video from youtubeDL
34 tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
35
36 // Get information about this video
37 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
38 const fps = await getVideoFileFPS(tempVideoPath)
39 const stats = await statPromise(tempVideoPath)
40 const duration = await getDurationFromVideoFile(tempVideoPath)
41
42 // Create video file object in database
43 const videoFileData = {
44 extname: extname(tempVideoPath),
45 resolution: videoFileResolution,
46 size: stats.size,
47 fps,
48 videoId: videoImport.videoId
49 }
50 const videoFile = new VideoFileModel(videoFileData)
51
52 // Move file
53 const destination = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
54 await renamePromise(tempVideoPath, destination)
55
56 // Process thumbnail
57 if (payload.downloadThumbnail) {
58 if (payload.thumbnailUrl) {
59 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
60 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
61 } else {
62 await videoImport.Video.createThumbnail(videoFile)
63 }
64 }
65
66 // Process preview
67 if (payload.downloadPreview) {
68 if (payload.thumbnailUrl) {
69 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
70 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
71 } else {
72 await videoImport.Video.createPreview(videoFile)
73 }
74 }
75
76 // Create torrent
77 await videoImport.Video.createTorrentAndSetInfoHash(videoFile)
78
79 const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => {
80 await videoFile.save({ transaction: t })
81
82 // Update video DB object
83 videoImport.Video.duration = duration
84 videoImport.Video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
85 const videoUpdated = await videoImport.Video.save({ transaction: t })
86
87 // Now we can federate the video
88 await federateVideoIfNeeded(videoImport.Video, true, t)
89
90 // Update video import object
91 videoImport.state = VideoImportState.SUCCESS
92 const videoImportUpdated = await videoImport.save({ transaction: t })
93
94 logger.info('Video %s imported.', videoImport.targetUrl)
95
96 videoImportUpdated.Video = videoUpdated
97 return videoImportUpdated
98 })
99
100 // Create transcoding jobs?
101 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
102 // Put uuid because we don't have id auto incremented for now
103 const dataInput = {
104 videoUUID: videoImportUpdated.Video.uuid,
105 isNewVideo: true
106 }
107
108 await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
109 }
110
111 } catch (err) {
112 try {
113 if (tempVideoPath) await unlinkPromise(tempVideoPath)
114 } catch (errUnlink) {
115 logger.error('Cannot cleanup files after a video import error.', { err: errUnlink })
116 }
117
118 videoImport.state = VideoImportState.FAILED
119 await videoImport.save()
120
121 throw err
122 }
123}
124
125// ---------------------------------------------------------------------------
126
127export {
128 processVideoImport
129}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8ff0c169e..2e14867f2 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -9,6 +9,7 @@ import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
12 13
13type CreateJobArgument = 14type CreateJobArgument =
14 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 15 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -17,7 +18,8 @@ type CreateJobArgument =
17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 18 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
18 { type: 'video-file-import', payload: VideoFileImportPayload } | 19 { type: 'video-file-import', payload: VideoFileImportPayload } |
19 { type: 'video-file', payload: VideoFilePayload } | 20 { type: 'video-file', payload: VideoFilePayload } |
20 { type: 'email', payload: EmailPayload } 21 { type: 'email', payload: EmailPayload } |
22 { type: 'video-import', payload: VideoImportPayload }
21 23
22const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { 24const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
23 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 25 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
@@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
26 'activitypub-follow': processActivityPubFollow, 28 'activitypub-follow': processActivityPubFollow,
27 'video-file-import': processVideoFileImport, 29 'video-file-import': processVideoFileImport,
28 'video-file': processVideoFile, 30 'video-file': processVideoFile,
29 'email': processEmail 31 'email': processEmail,
32 'video-import': processVideoImport
30} 33}
31 34
32const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { 35const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
@@ -43,7 +46,8 @@ const jobTypes: JobType[] = [
43 'activitypub-http-unicast', 46 'activitypub-http-unicast',
44 'email', 47 'email',
45 'video-file', 48 'video-file',
46 'video-file-import' 49 'video-file-import',
50 'video-import'
47] 51]
48 52
49class JobQueue { 53class JobQueue {
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index e3f0f5963..c5400c8f5 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -11,3 +11,4 @@ export * from './video-blacklist'
11export * from './video-channels' 11export * from './video-channels'
12export * from './webfinger' 12export * from './webfinger'
13export * from './search' 13export * from './search'
14export * from './video-imports'
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
new file mode 100644
index 000000000..0ba759ff0
--- /dev/null
+++ b/server/middlewares/validators/video-imports.ts
@@ -0,0 +1,51 @@
1import * as express from 'express'
2import { body, param } from 'express-validator/check'
3import { isIdValid } from '../../helpers/custom-validators/misc'
4import { logger } from '../../helpers/logger'
5import { areValidationErrors } from './utils'
6import { getCommonVideoAttributes } from './videos'
7import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
8import { cleanUpReqFiles } from '../../helpers/utils'
9import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'
10
11const videoImportAddValidator = getCommonVideoAttributes().concat([
12 body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
13 body('channelId')
14 .toInt()
15 .custom(isIdValid).withMessage('Should have correct video channel id'),
16 body('name')
17 .optional()
18 .custom(isVideoNameValid).withMessage('Should have a valid name'),
19
20 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
21 logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body })
22
23 const user = res.locals.oauth.token.User
24
25 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
26 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
27
28 return next()
29 }
30])
31
32const videoImportDeleteValidator = [
33 param('id').custom(isIdValid).not().isEmpty().withMessage('Should have a valid id'),
34
35 (req: express.Request, res: express.Response, next: express.NextFunction) => {
36 logger.debug('Checking videoImportDeleteValidator parameters', { parameters: req.body })
37
38 if (areValidationErrors(req, res)) return
39
40 return next()
41 }
42]
43
44// ---------------------------------------------------------------------------
45
46export {
47 videoImportAddValidator,
48 videoImportDeleteValidator
49}
50
51// ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts
index 9357c1e39..c812d4677 100644
--- a/server/middlewares/validators/videos.ts
+++ b/server/middlewares/validators/videos.ts
@@ -223,36 +223,6 @@ const videosShareValidator = [
223 } 223 }
224] 224]
225 225
226// ---------------------------------------------------------------------------
227
228export {
229 videosAddValidator,
230 videosUpdateValidator,
231 videosGetValidator,
232 videosRemoveValidator,
233 videosShareValidator,
234
235 videoAbuseReportValidator,
236
237 videoRateValidator
238}
239
240// ---------------------------------------------------------------------------
241
242function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) {
243 if (req.body.scheduleUpdate) {
244 if (!req.body.scheduleUpdate.updateAt) {
245 res.status(400)
246 .json({ error: 'Schedule update at is mandatory.' })
247 .end()
248
249 return true
250 }
251 }
252
253 return false
254}
255
256function getCommonVideoAttributes () { 226function getCommonVideoAttributes () {
257 return [ 227 return [
258 body('thumbnailfile') 228 body('thumbnailfile')
@@ -319,3 +289,35 @@ function getCommonVideoAttributes () {
319 .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy') 289 .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy')
320 ] as (ValidationChain | express.Handler)[] 290 ] as (ValidationChain | express.Handler)[]
321} 291}
292
293// ---------------------------------------------------------------------------
294
295export {
296 videosAddValidator,
297 videosUpdateValidator,
298 videosGetValidator,
299 videosRemoveValidator,
300 videosShareValidator,
301
302 videoAbuseReportValidator,
303
304 videoRateValidator,
305
306 getCommonVideoAttributes
307}
308
309// ---------------------------------------------------------------------------
310
311function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) {
312 if (req.body.scheduleUpdate) {
313 if (!req.body.scheduleUpdate.updateAt) {
314 res.status(400)
315 .json({ error: 'Schedule update at is mandatory.' })
316 .end()
317
318 return true
319 }
320 }
321
322 return false
323}
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index d674d8d22..66f5dcf2e 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -16,7 +16,6 @@ import {
16} from 'sequelize-typescript' 16} from 'sequelize-typescript'
17import { Account } from '../../../shared/models/actors' 17import { Account } from '../../../shared/models/actors'
18import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' 18import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts'
19import { logger } from '../../helpers/logger'
20import { sendDeleteActor } from '../../lib/activitypub/send' 19import { sendDeleteActor } from '../../lib/activitypub/send'
21import { ActorModel } from '../activitypub/actor' 20import { ActorModel } from '../activitypub/actor'
22import { ApplicationModel } from '../application/application' 21import { ApplicationModel } from '../application/application'
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
new file mode 100644
index 000000000..89eeafd6a
--- /dev/null
+++ b/server/models/video/video-import.ts
@@ -0,0 +1,105 @@
1import {
2 AllowNull,
3 BelongsTo,
4 Column,
5 CreatedAt,
6 DataType,
7 Default,
8 DefaultScope,
9 ForeignKey,
10 Is,
11 Model,
12 Table,
13 UpdatedAt
14} from 'sequelize-typescript'
15import { CONSTRAINTS_FIELDS } from '../../initializers'
16import { throwIfNotValid } from '../utils'
17import { VideoModel } from './video'
18import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
19import { VideoImport, VideoImportState } from '../../../shared'
20import { VideoChannelModel } from './video-channel'
21import { AccountModel } from '../account/account'
22
23@DefaultScope({
24 include: [
25 {
26 model: () => VideoModel,
27 required: true,
28 include: [
29 {
30 model: () => VideoChannelModel,
31 required: true,
32 include: [
33 {
34 model: () => AccountModel,
35 required: true
36 }
37 ]
38 }
39 ]
40 }
41 ]
42})
43
44@Table({
45 tableName: 'videoImport',
46 indexes: [
47 {
48 fields: [ 'videoId' ],
49 unique: true
50 }
51 ]
52})
53export class VideoImportModel extends Model<VideoImportModel> {
54 @CreatedAt
55 createdAt: Date
56
57 @UpdatedAt
58 updatedAt: Date
59
60 @AllowNull(false)
61 @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
62 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
63 targetUrl: string
64
65 @AllowNull(false)
66 @Default(null)
67 @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))
68 @Column
69 state: VideoImportState
70
71 @AllowNull(true)
72 @Default(null)
73 @Column(DataType.TEXT)
74 error: string
75
76 @ForeignKey(() => VideoModel)
77 @Column
78 videoId: number
79
80 @BelongsTo(() => VideoModel, {
81 foreignKey: {
82 allowNull: false
83 },
84 onDelete: 'CASCADE'
85 })
86 Video: VideoModel
87
88 static loadAndPopulateVideo (id: number) {
89 return VideoImportModel.findById(id)
90 }
91
92 toFormattedJSON (): VideoImport {
93 const videoFormatOptions = {
94 additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true }
95 }
96 const video = Object.assign(this.Video.toFormattedJSON(videoFormatOptions), {
97 tags: this.Video.Tags.map(t => t.name)
98 })
99
100 return {
101 targetUrl: this.targetUrl,
102 video
103 }
104 }
105}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index a6c4620b2..459fcb31e 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -377,7 +377,7 @@ type AvailableForListOptions = {
377 include: [ 377 include: [
378 { 378 {
379 model: () => VideoFileModel.unscoped(), 379 model: () => VideoFileModel.unscoped(),
380 required: true 380 required: false
381 } 381 }
382 ] 382 ]
383 }, 383 },