aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-06 17:13:39 +0200
committerChocobozzz <me@florianbigard.com>2018-08-08 09:30:31 +0200
commitce33919c24e7402d92d81f3cd8e545df52d98240 (patch)
tree7e131a2f8df649899d0a71294665cf386ffb50d4 /server
parent788487140c500abeb69ca44daf3a9e26efa8d36f (diff)
downloadPeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.gz
PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.zst
PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.zip
Import magnets with webtorrent
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/videos/import.ts142
-rw-r--r--server/helpers/custom-validators/videos.ts9
-rw-r--r--server/helpers/utils.ts11
-rw-r--r--server/helpers/webtorrent.ts31
-rw-r--r--server/helpers/youtube-dl.ts24
-rw-r--r--server/initializers/constants.ts5
-rw-r--r--server/initializers/migrations/0245-import-magnet.ts42
-rw-r--r--server/lib/job-queue/handlers/video-import.ts99
-rw-r--r--server/lib/job-queue/job-queue.ts7
-rw-r--r--server/middlewares/validators/video-imports.ts18
-rw-r--r--server/models/video/video-import.ts15
11 files changed, 324 insertions, 79 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
index 30a7d816c..c16a254d2 100644
--- a/server/controllers/api/videos/import.ts
+++ b/server/controllers/api/videos/import.ts
@@ -1,3 +1,4 @@
1import * as magnetUtil from 'magnet-uri'
1import * as express from 'express' 2import * as express from 'express'
2import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' 3import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
3import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' 4import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
@@ -13,6 +14,10 @@ import { VideoImportModel } from '../../../models/video/video-import'
13import { JobQueue } from '../../../lib/job-queue/job-queue' 14import { JobQueue } from '../../../lib/job-queue/job-queue'
14import { processImage } from '../../../helpers/image-utils' 15import { processImage } from '../../../helpers/image-utils'
15import { join } from 'path' 16import { join } from 'path'
17import { isArray } from '../../../helpers/custom-validators/misc'
18import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
19import { VideoChannelModel } from '../../../models/video/video-channel'
20import * as Bluebird from 'bluebird'
16 21
17const auditLogger = auditLoggerFactory('video-imports') 22const auditLogger = auditLoggerFactory('video-imports')
18const videoImportsRouter = express.Router() 23const videoImportsRouter = express.Router()
@@ -41,7 +46,45 @@ export {
41 46
42// --------------------------------------------------------------------------- 47// ---------------------------------------------------------------------------
43 48
44async function addVideoImport (req: express.Request, res: express.Response) { 49function addVideoImport (req: express.Request, res: express.Response) {
50 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
51
52 if (req.body.magnetUri) return addTorrentImport(req, res)
53}
54
55async function addTorrentImport (req: express.Request, res: express.Response) {
56 const body: VideoImportCreate = req.body
57 const magnetUri = body.magnetUri
58
59 const parsed = magnetUtil.decode(magnetUri)
60 const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string
61
62 const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName })
63
64 await processThumbnail(req, video)
65 await processPreview(req, video)
66
67 const tags = null
68 const videoImportAttributes = {
69 magnetUri,
70 state: VideoImportState.PENDING
71 }
72 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
73
74 // Create job to import the video
75 const payload = {
76 type: 'magnet-uri' as 'magnet-uri',
77 videoImportId: videoImport.id,
78 magnetUri
79 }
80 await JobQueue.Instance.createJob({ type: 'video-import', payload })
81
82 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
83
84 return res.json(videoImport.toFormattedJSON()).end()
85}
86
87async function addYoutubeDLImport (req: express.Request, res: express.Response) {
45 const body: VideoImportCreate = req.body 88 const body: VideoImportCreate = req.body
46 const targetUrl = body.targetUrl 89 const targetUrl = body.targetUrl
47 90
@@ -56,53 +99,94 @@ async function addVideoImport (req: express.Request, res: express.Response) {
56 }).end() 99 }).end()
57 } 100 }
58 101
59 // Create video DB object 102 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
103
104 const downloadThumbnail = !await processThumbnail(req, video)
105 const downloadPreview = !await processPreview(req, video)
106
107 const tags = body.tags || youtubeDLInfo.tags
108 const videoImportAttributes = {
109 targetUrl,
110 state: VideoImportState.PENDING
111 }
112 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
113
114 // Create job to import the video
115 const payload = {
116 type: 'youtube-dl' as 'youtube-dl',
117 videoImportId: videoImport.id,
118 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
119 downloadThumbnail,
120 downloadPreview
121 }
122 await JobQueue.Instance.createJob({ type: 'video-import', payload })
123
124 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
125
126 return res.json(videoImport.toFormattedJSON()).end()
127}
128
129function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
60 const videoData = { 130 const videoData = {
61 name: body.name || youtubeDLInfo.name, 131 name: body.name || importData.name || 'Unknown name',
62 remote: false, 132 remote: false,
63 category: body.category || youtubeDLInfo.category, 133 category: body.category || importData.category,
64 licence: body.licence || youtubeDLInfo.licence, 134 licence: body.licence || importData.licence,
65 language: body.language || undefined, 135 language: body.language || undefined,
66 commentsEnabled: body.commentsEnabled || true, 136 commentsEnabled: body.commentsEnabled || true,
67 waitTranscoding: body.waitTranscoding || false, 137 waitTranscoding: body.waitTranscoding || false,
68 state: VideoState.TO_IMPORT, 138 state: VideoState.TO_IMPORT,
69 nsfw: body.nsfw || youtubeDLInfo.nsfw || false, 139 nsfw: body.nsfw || importData.nsfw || false,
70 description: body.description || youtubeDLInfo.description, 140 description: body.description || importData.description,
71 support: body.support || null, 141 support: body.support || null,
72 privacy: body.privacy || VideoPrivacy.PRIVATE, 142 privacy: body.privacy || VideoPrivacy.PRIVATE,
73 duration: 0, // duration will be set by the import job 143 duration: 0, // duration will be set by the import job
74 channelId: res.locals.videoChannel.id 144 channelId: channelId
75 } 145 }
76 const video = new VideoModel(videoData) 146 const video = new VideoModel(videoData)
77 video.url = getVideoActivityPubUrl(video) 147 video.url = getVideoActivityPubUrl(video)
78 148
79 // Process thumbnail file? 149 return video
150}
151
152async function processThumbnail (req: express.Request, video: VideoModel) {
80 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined 153 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
81 let downloadThumbnail = true
82 if (thumbnailField) { 154 if (thumbnailField) {
83 const thumbnailPhysicalFile = thumbnailField[ 0 ] 155 const thumbnailPhysicalFile = thumbnailField[ 0 ]
84 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) 156 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
85 downloadThumbnail = false 157
158 return true
86 } 159 }
87 160
88 // Process preview file? 161 return false
162}
163
164async function processPreview (req: express.Request, video: VideoModel) {
89 const previewField = req.files ? req.files['previewfile'] : undefined 165 const previewField = req.files ? req.files['previewfile'] : undefined
90 let downloadPreview = true
91 if (previewField) { 166 if (previewField) {
92 const previewPhysicalFile = previewField[0] 167 const previewPhysicalFile = previewField[0]
93 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) 168 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
94 downloadPreview = false 169
170 return true
95 } 171 }
96 172
97 const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { 173 return false
174}
175
176function insertIntoDB (
177 video: VideoModel,
178 videoChannel: VideoChannelModel,
179 tags: string[],
180 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
181): Bluebird<VideoImportModel> {
182 return sequelizeTypescript.transaction(async t => {
98 const sequelizeOptions = { transaction: t } 183 const sequelizeOptions = { transaction: t }
99 184
100 // Save video object in database 185 // Save video object in database
101 const videoCreated = await video.save(sequelizeOptions) 186 const videoCreated = await video.save(sequelizeOptions)
102 videoCreated.VideoChannel = res.locals.videoChannel 187 videoCreated.VideoChannel = videoChannel
103 188
104 // Set tags to the video 189 // Set tags to the video
105 const tags = body.tags ? body.tags : youtubeDLInfo.tags
106 if (tags !== undefined) { 190 if (tags !== undefined) {
107 const tagInstances = await TagModel.findOrCreateTags(tags, t) 191 const tagInstances = await TagModel.findOrCreateTags(tags, t)
108 192
@@ -111,28 +195,12 @@ async function addVideoImport (req: express.Request, res: express.Response) {
111 } 195 }
112 196
113 // Create video import object in database 197 // Create video import object in database
114 const videoImport = await VideoImportModel.create({ 198 const videoImport = await VideoImportModel.create(
115 targetUrl, 199 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
116 state: VideoImportState.PENDING, 200 sequelizeOptions
117 videoId: videoCreated.id 201 )
118 }, sequelizeOptions)
119
120 videoImport.Video = videoCreated 202 videoImport.Video = videoCreated
121 203
122 return videoImport 204 return videoImport
123 }) 205 })
124
125 // Create job to import the video
126 const payload = {
127 type: 'youtube-dl' as 'youtube-dl',
128 videoImportId: videoImport.id,
129 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
130 downloadThumbnail,
131 downloadPreview
132 }
133 await JobQueue.Instance.createJob({ type: 'video-import', payload })
134
135 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
136
137 return res.json(videoImport.toFormattedJSON()).end()
138} 206}
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 338c96582..f4c1c8b07 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -17,6 +17,7 @@ import { VideoModel } from '../../models/video/video'
17import { exists, isArray, isFileValid } from './misc' 17import { exists, isArray, isFileValid } from './misc'
18import { VideoChannelModel } from '../../models/video/video-channel' 18import { VideoChannelModel } from '../../models/video/video-channel'
19import { UserModel } from '../../models/account/user' 19import { UserModel } from '../../models/account/user'
20import * as magnetUtil from 'magnet-uri'
20 21
21const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS 22const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
22const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES 23const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES
@@ -126,6 +127,13 @@ function isVideoFileSizeValid (value: string) {
126 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) 127 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
127} 128}
128 129
130function isVideoMagnetUriValid (value: string) {
131 if (!exists(value)) return false
132
133 const parsed = magnetUtil.decode(value)
134 return parsed && isVideoFileInfoHashValid(parsed.infoHash)
135}
136
129function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) { 137function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) {
130 // Retrieve the user who did the request 138 // Retrieve the user who did the request
131 if (video.isOwned() === false) { 139 if (video.isOwned() === false) {
@@ -214,6 +222,7 @@ export {
214 isScheduleVideoUpdatePrivacyValid, 222 isScheduleVideoUpdatePrivacyValid,
215 isVideoAbuseReasonValid, 223 isVideoAbuseReasonValid,
216 isVideoFile, 224 isVideoFile,
225 isVideoMagnetUriValid,
217 isVideoStateValid, 226 isVideoStateValid,
218 isVideoViewsValid, 227 isVideoViewsValid,
219 isVideoRatingTypeValid, 228 isVideoRatingTypeValid,
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 7abcec5d7..f4cc5547d 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -9,6 +9,8 @@ import { ApplicationModel } from '../models/application/application'
9import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' 9import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils'
10import { logger } from './logger' 10import { logger } from './logger'
11import { isArray } from './custom-validators/misc' 11import { isArray } from './custom-validators/misc'
12import * as crypto from "crypto"
13import { join } from "path"
12 14
13const isCidr = require('is-cidr') 15const isCidr = require('is-cidr')
14 16
@@ -181,8 +183,14 @@ async function getServerActor () {
181 return Promise.resolve(serverActor) 183 return Promise.resolve(serverActor)
182} 184}
183 185
186function generateVideoTmpPath (id: string) {
187 const hash = crypto.createHash('sha256').update(id).digest('hex')
188 return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
189}
190
184type SortType = { sortModel: any, sortValue: string } 191type SortType = { sortModel: any, sortValue: string }
185 192
193
186// --------------------------------------------------------------------------- 194// ---------------------------------------------------------------------------
187 195
188export { 196export {
@@ -195,5 +203,6 @@ export {
195 computeResolutionsToTranscode, 203 computeResolutionsToTranscode,
196 resetSequelizeInstance, 204 resetSequelizeInstance,
197 getServerActor, 205 getServerActor,
198 SortType 206 SortType,
207 generateVideoTmpPath
199} 208}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
new file mode 100644
index 000000000..fce88a1f6
--- /dev/null
+++ b/server/helpers/webtorrent.ts
@@ -0,0 +1,31 @@
1import { logger } from './logger'
2import { generateVideoTmpPath } from './utils'
3import * as WebTorrent from 'webtorrent'
4import { createWriteStream } from 'fs'
5
6function downloadWebTorrentVideo (target: string) {
7 const path = generateVideoTmpPath(target)
8
9 logger.info('Importing torrent video %s', target)
10
11 return new Promise<string>((res, rej) => {
12 const webtorrent = new WebTorrent()
13
14 const torrent = webtorrent.add(target, torrent => {
15 if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target)
16
17 const file = torrent.files[ 0 ]
18 file.createReadStream().pipe(createWriteStream(path))
19 })
20
21 torrent.on('done', () => res(path))
22
23 torrent.on('error', err => rej(err))
24 })
25}
26
27// ---------------------------------------------------------------------------
28
29export {
30 downloadWebTorrentVideo
31}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index c59ab9de0..77986f407 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,18 +1,17 @@
1import * as youtubeDL from 'youtube-dl' 1import * as youtubeDL from 'youtube-dl'
2import { truncate } from 'lodash' 2import { truncate } from 'lodash'
3import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' 3import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
4import { join } from 'path'
5import * as crypto from 'crypto'
6import { logger } from './logger' 4import { logger } from './logger'
5import { generateVideoTmpPath } from './utils'
7 6
8export type YoutubeDLInfo = { 7export type YoutubeDLInfo = {
9 name: string 8 name?: string
10 description: string 9 description?: string
11 category: number 10 category?: number
12 licence: number 11 licence?: number
13 nsfw: boolean 12 nsfw?: boolean
14 tags: string[] 13 tags?: string[]
15 thumbnailUrl: string 14 thumbnailUrl?: string
16} 15}
17 16
18function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> { 17function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
@@ -30,10 +29,9 @@ function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
30} 29}
31 30
32function downloadYoutubeDLVideo (url: string) { 31function downloadYoutubeDLVideo (url: string) {
33 const hash = crypto.createHash('sha256').update(url).digest('hex') 32 const path = generateVideoTmpPath(url)
34 const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
35 33
36 logger.info('Importing video %s', url) 34 logger.info('Importing youtubeDL video %s', url)
37 35
38 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] 36 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
39 37
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 74fe7965d..243d544ea 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -15,7 +15,7 @@ let config: IConfig = require('config')
15 15
16// --------------------------------------------------------------------------- 16// ---------------------------------------------------------------------------
17 17
18const LAST_MIGRATION_VERSION = 240 18const LAST_MIGRATION_VERSION = 245
19 19
20// --------------------------------------------------------------------------- 20// ---------------------------------------------------------------------------
21 21
@@ -271,7 +271,8 @@ const CONSTRAINTS_FIELDS = {
271 } 271 }
272 }, 272 },
273 VIDEO_IMPORTS: { 273 VIDEO_IMPORTS: {
274 URL: { min: 3, max: 2000 } // Length 274 URL: { min: 3, max: 2000 }, // Length
275 TORRENT_NAME: { min: 3, max: 255 }, // Length
275 }, 276 },
276 VIDEOS: { 277 VIDEOS: {
277 NAME: { min: 3, max: 120 }, // Length 278 NAME: { min: 3, max: 120 }, // Length
diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts
new file mode 100644
index 000000000..87603b006
--- /dev/null
+++ b/server/initializers/migrations/0245-import-magnet.ts
@@ -0,0 +1,42 @@
1import * as Sequelize from 'sequelize'
2import { Migration } from '../../models/migrations'
3import { CONSTRAINTS_FIELDS } from '../index'
4
5async function up (utils: {
6 transaction: Sequelize.Transaction
7 queryInterface: Sequelize.QueryInterface
8 sequelize: Sequelize.Sequelize
9}): Promise<any> {
10 {
11 const data = {
12 type: Sequelize.STRING,
13 allowNull: true,
14 defaultValue: null
15 } as Migration.String
16 await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data)
17 }
18
19 {
20 const data = {
21 type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max),
22 allowNull: true,
23 defaultValue: null
24 }
25 await utils.queryInterface.addColumn('videoImport', 'magnetUri', data)
26 }
27
28 {
29 const data = {
30 type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max),
31 allowNull: true,
32 defaultValue: null
33 }
34 await utils.queryInterface.addColumn('videoImport', 'torrentName', data)
35 }
36}
37
38function down (options) {
39 throw new Error('Not implemented.')
40}
41
42export { up, down }
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index cdfe412cc..c457b71fc 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared'
13import { JobQueue } from '../index' 13import { JobQueue } from '../index'
14import { federateVideoIfNeeded } from '../../activitypub' 14import { federateVideoIfNeeded } from '../../activitypub'
15import { VideoModel } from '../../../models/video/video' 15import { VideoModel } from '../../../models/video/video'
16import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
16 17
17export type VideoImportPayload = { 18type VideoImportYoutubeDLPayload = {
18 type: 'youtube-dl' 19 type: 'youtube-dl'
19 videoImportId: number 20 videoImportId: number
21
20 thumbnailUrl: string 22 thumbnailUrl: string
21 downloadThumbnail: boolean 23 downloadThumbnail: boolean
22 downloadPreview: boolean 24 downloadPreview: boolean
23} 25}
24 26
27type VideoImportTorrentPayload = {
28 type: 'magnet-uri'
29 videoImportId: number
30}
31
32export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
33
25async function processVideoImport (job: Bull.Job) { 34async function processVideoImport (job: Bull.Job) {
26 const payload = job.data as VideoImportPayload 35 const payload = job.data as VideoImportPayload
27 logger.info('Processing video import in job %d.', job.id)
28 36
29 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) 37 if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
38 if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
39}
40
41// ---------------------------------------------------------------------------
42
43export {
44 processVideoImport
45}
46
47// ---------------------------------------------------------------------------
48
49async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
50 logger.info('Processing torrent video import in job %d.', job.id)
51
52 const videoImport = await getVideoImportOrDie(payload.videoImportId)
53 const options = {
54 videoImportId: payload.videoImportId,
55
56 downloadThumbnail: false,
57 downloadPreview: false,
58
59 generateThumbnail: true,
60 generatePreview: true
61 }
62 return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
63}
64
65async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
66 logger.info('Processing youtubeDL video import in job %d.', job.id)
67
68 const videoImport = await getVideoImportOrDie(payload.videoImportId)
69 const options = {
70 videoImportId: videoImport.id,
71
72 downloadThumbnail: payload.downloadThumbnail,
73 downloadPreview: payload.downloadPreview,
74 thumbnailUrl: payload.thumbnailUrl,
75
76 generateThumbnail: false,
77 generatePreview: false
78 }
79
80 return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
81}
82
83async function getVideoImportOrDie (videoImportId: number) {
84 const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
30 if (!videoImport || !videoImport.Video) { 85 if (!videoImport || !videoImport.Video) {
31 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') 86 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
32 } 87 }
33 88
89 return videoImport
90}
91
92type ProcessFileOptions = {
93 videoImportId: number
94
95 downloadThumbnail: boolean
96 downloadPreview: boolean
97 thumbnailUrl?: string
98
99 generateThumbnail: boolean
100 generatePreview: boolean
101}
102async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
34 let tempVideoPath: string 103 let tempVideoPath: string
35 let videoDestFile: string 104 let videoDestFile: string
36 let videoFile: VideoFileModel 105 let videoFile: VideoFileModel
37 try { 106 try {
38 // Download video from youtubeDL 107 // Download video from youtubeDL
39 tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) 108 tempVideoPath = await downloader()
40 109
41 // Get information about this video 110 // Get information about this video
42 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) 111 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
@@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) {
62 tempVideoPath = null // This path is not used anymore 131 tempVideoPath = null // This path is not used anymore
63 132
64 // Process thumbnail 133 // Process thumbnail
65 if (payload.downloadThumbnail) { 134 if (options.downloadThumbnail) {
66 if (payload.thumbnailUrl) { 135 if (options.thumbnailUrl) {
67 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) 136 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
68 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) 137 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
69 } else { 138 } else {
70 await videoImport.Video.createThumbnail(videoFile) 139 await videoImport.Video.createThumbnail(videoFile)
71 } 140 }
141 } else if (options.generateThumbnail) {
142 await videoImport.Video.createThumbnail(videoFile)
72 } 143 }
73 144
74 // Process preview 145 // Process preview
75 if (payload.downloadPreview) { 146 if (options.downloadPreview) {
76 if (payload.thumbnailUrl) { 147 if (options.thumbnailUrl) {
77 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) 148 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
78 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) 149 await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
79 } else { 150 } else {
80 await videoImport.Video.createPreview(videoFile) 151 await videoImport.Video.createPreview(videoFile)
81 } 152 }
153 } else if (options.generatePreview) {
154 await videoImport.Video.createPreview(videoFile)
82 } 155 }
83 156
84 // Create torrent 157 // Create torrent
@@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) {
137 throw err 210 throw err
138 } 211 }
139} 212}
140
141// ---------------------------------------------------------------------------
142
143export {
144 processVideoImport
145}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8a24604e1..ddb357db5 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
32 'video-import': processVideoImport 32 'video-import': processVideoImport
33} 33}
34 34
35const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
36 'activitypub-http-broadcast': true,
37 'activitypub-http-unicast': true,
38 'activitypub-http-fetcher': true,
39 'activitypub-follow': true
40}
41
42const jobTypes: JobType[] = [ 35const jobTypes: JobType[] = [
43 'activitypub-follow', 36 'activitypub-follow',
44 'activitypub-http-broadcast', 37 'activitypub-http-broadcast',
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
index d806edfa3..8ec9373fb 100644
--- a/server/middlewares/validators/video-imports.ts
+++ b/server/middlewares/validators/video-imports.ts
@@ -6,14 +6,19 @@ import { areValidationErrors } from './utils'
6import { getCommonVideoAttributes } from './videos' 6import { getCommonVideoAttributes } from './videos'
7import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' 7import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
8import { cleanUpReqFiles } from '../../helpers/utils' 8import { cleanUpReqFiles } from '../../helpers/utils'
9import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' 9import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos'
10import { CONFIG } from '../../initializers/constants' 10import { CONFIG } from '../../initializers/constants'
11 11
12const videoImportAddValidator = getCommonVideoAttributes().concat([ 12const videoImportAddValidator = getCommonVideoAttributes().concat([
13 body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
14 body('channelId') 13 body('channelId')
15 .toInt() 14 .toInt()
16 .custom(isIdValid).withMessage('Should have correct video channel id'), 15 .custom(isIdValid).withMessage('Should have correct video channel id'),
16 body('targetUrl')
17 .optional()
18 .custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
19 body('magnetUri')
20 .optional()
21 .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'),
17 body('name') 22 body('name')
18 .optional() 23 .optional()
19 .custom(isVideoNameValid).withMessage('Should have a valid name'), 24 .custom(isVideoNameValid).withMessage('Should have a valid name'),
@@ -34,6 +39,15 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
34 39
35 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) 40 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
36 41
42 // Check we have at least 1 required param
43 if (!req.body.targetUrl && !req.body.magnetUri) {
44 cleanUpReqFiles(req)
45
46 return res.status(400)
47 .json({ error: 'Should have a magnetUri or a targetUrl.' })
48 .end()
49 }
50
37 return next() 51 return next()
38 } 52 }
39]) 53])
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
index eca87163d..55fca28b8 100644
--- a/server/models/video/video-import.ts
+++ b/server/models/video/video-import.ts
@@ -21,6 +21,7 @@ import { VideoImport, VideoImportState } from '../../../shared'
21import { VideoChannelModel } from './video-channel' 21import { VideoChannelModel } from './video-channel'
22import { AccountModel } from '../account/account' 22import { AccountModel } from '../account/account'
23import { TagModel } from './tag' 23import { TagModel } from './tag'
24import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
24 25
25@DefaultScope({ 26@DefaultScope({
26 include: [ 27 include: [
@@ -62,11 +63,23 @@ export class VideoImportModel extends Model<VideoImportModel> {
62 @UpdatedAt 63 @UpdatedAt
63 updatedAt: Date 64 updatedAt: Date
64 65
65 @AllowNull(false) 66 @AllowNull(true)
67 @Default(null)
66 @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) 68 @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
67 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) 69 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
68 targetUrl: string 70 targetUrl: string
69 71
72 @AllowNull(true)
73 @Default(null)
74 @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri'))
75 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs
76 magnetUri: string
77
78 @AllowNull(true)
79 @Default(null)
80 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max))
81 torrentName: string
82
70 @AllowNull(false) 83 @AllowNull(false)
71 @Default(null) 84 @Default(null)
72 @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) 85 @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))