diff options
author | Chocobozzz <me@florianbigard.com> | 2018-08-06 17:13:39 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-08-08 09:30:31 +0200 |
commit | ce33919c24e7402d92d81f3cd8e545df52d98240 (patch) | |
tree | 7e131a2f8df649899d0a71294665cf386ffb50d4 /server | |
parent | 788487140c500abeb69ca44daf3a9e26efa8d36f (diff) | |
download | PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.gz PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.tar.zst PeerTube-ce33919c24e7402d92d81f3cd8e545df52d98240.zip |
Import magnets with webtorrent
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/videos/import.ts | 142 | ||||
-rw-r--r-- | server/helpers/custom-validators/videos.ts | 9 | ||||
-rw-r--r-- | server/helpers/utils.ts | 11 | ||||
-rw-r--r-- | server/helpers/webtorrent.ts | 31 | ||||
-rw-r--r-- | server/helpers/youtube-dl.ts | 24 | ||||
-rw-r--r-- | server/initializers/constants.ts | 5 | ||||
-rw-r--r-- | server/initializers/migrations/0245-import-magnet.ts | 42 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-import.ts | 99 | ||||
-rw-r--r-- | server/lib/job-queue/job-queue.ts | 7 | ||||
-rw-r--r-- | server/middlewares/validators/video-imports.ts | 18 | ||||
-rw-r--r-- | server/models/video/video-import.ts | 15 |
11 files changed, 324 insertions, 79 deletions
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 30a7d816c..c16a254d2 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import * as magnetUtil from 'magnet-uri' | ||
1 | import * as express from 'express' | 2 | import * as express from 'express' |
2 | import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' | 3 | import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' |
3 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' | 4 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' |
@@ -13,6 +14,10 @@ import { VideoImportModel } from '../../../models/video/video-import' | |||
13 | import { JobQueue } from '../../../lib/job-queue/job-queue' | 14 | import { JobQueue } from '../../../lib/job-queue/job-queue' |
14 | import { processImage } from '../../../helpers/image-utils' | 15 | import { processImage } from '../../../helpers/image-utils' |
15 | import { join } from 'path' | 16 | import { join } from 'path' |
17 | import { isArray } from '../../../helpers/custom-validators/misc' | ||
18 | import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' | ||
19 | import { VideoChannelModel } from '../../../models/video/video-channel' | ||
20 | import * as Bluebird from 'bluebird' | ||
16 | 21 | ||
17 | const auditLogger = auditLoggerFactory('video-imports') | 22 | const auditLogger = auditLoggerFactory('video-imports') |
18 | const videoImportsRouter = express.Router() | 23 | const videoImportsRouter = express.Router() |
@@ -41,7 +46,45 @@ export { | |||
41 | 46 | ||
42 | // --------------------------------------------------------------------------- | 47 | // --------------------------------------------------------------------------- |
43 | 48 | ||
44 | async function addVideoImport (req: express.Request, res: express.Response) { | 49 | function addVideoImport (req: express.Request, res: express.Response) { |
50 | if (req.body.targetUrl) return addYoutubeDLImport(req, res) | ||
51 | |||
52 | if (req.body.magnetUri) return addTorrentImport(req, res) | ||
53 | } | ||
54 | |||
55 | async function addTorrentImport (req: express.Request, res: express.Response) { | ||
56 | const body: VideoImportCreate = req.body | ||
57 | const magnetUri = body.magnetUri | ||
58 | |||
59 | const parsed = magnetUtil.decode(magnetUri) | ||
60 | const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string | ||
61 | |||
62 | const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName }) | ||
63 | |||
64 | await processThumbnail(req, video) | ||
65 | await processPreview(req, video) | ||
66 | |||
67 | const tags = null | ||
68 | const videoImportAttributes = { | ||
69 | magnetUri, | ||
70 | state: VideoImportState.PENDING | ||
71 | } | ||
72 | const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) | ||
73 | |||
74 | // Create job to import the video | ||
75 | const payload = { | ||
76 | type: 'magnet-uri' as 'magnet-uri', | ||
77 | videoImportId: videoImport.id, | ||
78 | magnetUri | ||
79 | } | ||
80 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | ||
81 | |||
82 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) | ||
83 | |||
84 | return res.json(videoImport.toFormattedJSON()).end() | ||
85 | } | ||
86 | |||
87 | async function addYoutubeDLImport (req: express.Request, res: express.Response) { | ||
45 | const body: VideoImportCreate = req.body | 88 | const body: VideoImportCreate = req.body |
46 | const targetUrl = body.targetUrl | 89 | const targetUrl = body.targetUrl |
47 | 90 | ||
@@ -56,53 +99,94 @@ async function addVideoImport (req: express.Request, res: express.Response) { | |||
56 | }).end() | 99 | }).end() |
57 | } | 100 | } |
58 | 101 | ||
59 | // Create video DB object | 102 | const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo) |
103 | |||
104 | const downloadThumbnail = !await processThumbnail(req, video) | ||
105 | const downloadPreview = !await processPreview(req, video) | ||
106 | |||
107 | const tags = body.tags || youtubeDLInfo.tags | ||
108 | const videoImportAttributes = { | ||
109 | targetUrl, | ||
110 | state: VideoImportState.PENDING | ||
111 | } | ||
112 | const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) | ||
113 | |||
114 | // Create job to import the video | ||
115 | const payload = { | ||
116 | type: 'youtube-dl' as 'youtube-dl', | ||
117 | videoImportId: videoImport.id, | ||
118 | thumbnailUrl: youtubeDLInfo.thumbnailUrl, | ||
119 | downloadThumbnail, | ||
120 | downloadPreview | ||
121 | } | ||
122 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | ||
123 | |||
124 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) | ||
125 | |||
126 | return res.json(videoImport.toFormattedJSON()).end() | ||
127 | } | ||
128 | |||
129 | function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) { | ||
60 | const videoData = { | 130 | const videoData = { |
61 | name: body.name || youtubeDLInfo.name, | 131 | name: body.name || importData.name || 'Unknown name', |
62 | remote: false, | 132 | remote: false, |
63 | category: body.category || youtubeDLInfo.category, | 133 | category: body.category || importData.category, |
64 | licence: body.licence || youtubeDLInfo.licence, | 134 | licence: body.licence || importData.licence, |
65 | language: body.language || undefined, | 135 | language: body.language || undefined, |
66 | commentsEnabled: body.commentsEnabled || true, | 136 | commentsEnabled: body.commentsEnabled || true, |
67 | waitTranscoding: body.waitTranscoding || false, | 137 | waitTranscoding: body.waitTranscoding || false, |
68 | state: VideoState.TO_IMPORT, | 138 | state: VideoState.TO_IMPORT, |
69 | nsfw: body.nsfw || youtubeDLInfo.nsfw || false, | 139 | nsfw: body.nsfw || importData.nsfw || false, |
70 | description: body.description || youtubeDLInfo.description, | 140 | description: body.description || importData.description, |
71 | support: body.support || null, | 141 | support: body.support || null, |
72 | privacy: body.privacy || VideoPrivacy.PRIVATE, | 142 | privacy: body.privacy || VideoPrivacy.PRIVATE, |
73 | duration: 0, // duration will be set by the import job | 143 | duration: 0, // duration will be set by the import job |
74 | channelId: res.locals.videoChannel.id | 144 | channelId: channelId |
75 | } | 145 | } |
76 | const video = new VideoModel(videoData) | 146 | const video = new VideoModel(videoData) |
77 | video.url = getVideoActivityPubUrl(video) | 147 | video.url = getVideoActivityPubUrl(video) |
78 | 148 | ||
79 | // Process thumbnail file? | 149 | return video |
150 | } | ||
151 | |||
152 | async function processThumbnail (req: express.Request, video: VideoModel) { | ||
80 | const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined | 153 | const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined |
81 | let downloadThumbnail = true | ||
82 | if (thumbnailField) { | 154 | if (thumbnailField) { |
83 | const thumbnailPhysicalFile = thumbnailField[ 0 ] | 155 | const thumbnailPhysicalFile = thumbnailField[ 0 ] |
84 | await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) | 156 | await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) |
85 | downloadThumbnail = false | 157 | |
158 | return true | ||
86 | } | 159 | } |
87 | 160 | ||
88 | // Process preview file? | 161 | return false |
162 | } | ||
163 | |||
164 | async function processPreview (req: express.Request, video: VideoModel) { | ||
89 | const previewField = req.files ? req.files['previewfile'] : undefined | 165 | const previewField = req.files ? req.files['previewfile'] : undefined |
90 | let downloadPreview = true | ||
91 | if (previewField) { | 166 | if (previewField) { |
92 | const previewPhysicalFile = previewField[0] | 167 | const previewPhysicalFile = previewField[0] |
93 | await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) | 168 | await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) |
94 | downloadPreview = false | 169 | |
170 | return true | ||
95 | } | 171 | } |
96 | 172 | ||
97 | const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { | 173 | return false |
174 | } | ||
175 | |||
176 | function insertIntoDB ( | ||
177 | video: VideoModel, | ||
178 | videoChannel: VideoChannelModel, | ||
179 | tags: string[], | ||
180 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | ||
181 | ): Bluebird<VideoImportModel> { | ||
182 | return sequelizeTypescript.transaction(async t => { | ||
98 | const sequelizeOptions = { transaction: t } | 183 | const sequelizeOptions = { transaction: t } |
99 | 184 | ||
100 | // Save video object in database | 185 | // Save video object in database |
101 | const videoCreated = await video.save(sequelizeOptions) | 186 | const videoCreated = await video.save(sequelizeOptions) |
102 | videoCreated.VideoChannel = res.locals.videoChannel | 187 | videoCreated.VideoChannel = videoChannel |
103 | 188 | ||
104 | // Set tags to the video | 189 | // Set tags to the video |
105 | const tags = body.tags ? body.tags : youtubeDLInfo.tags | ||
106 | if (tags !== undefined) { | 190 | if (tags !== undefined) { |
107 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | 191 | const tagInstances = await TagModel.findOrCreateTags(tags, t) |
108 | 192 | ||
@@ -111,28 +195,12 @@ async function addVideoImport (req: express.Request, res: express.Response) { | |||
111 | } | 195 | } |
112 | 196 | ||
113 | // Create video import object in database | 197 | // Create video import object in database |
114 | const videoImport = await VideoImportModel.create({ | 198 | const videoImport = await VideoImportModel.create( |
115 | targetUrl, | 199 | Object.assign({ videoId: videoCreated.id }, videoImportAttributes), |
116 | state: VideoImportState.PENDING, | 200 | sequelizeOptions |
117 | videoId: videoCreated.id | 201 | ) |
118 | }, sequelizeOptions) | ||
119 | |||
120 | videoImport.Video = videoCreated | 202 | videoImport.Video = videoCreated |
121 | 203 | ||
122 | return videoImport | 204 | return videoImport |
123 | }) | 205 | }) |
124 | |||
125 | // Create job to import the video | ||
126 | const payload = { | ||
127 | type: 'youtube-dl' as 'youtube-dl', | ||
128 | videoImportId: videoImport.id, | ||
129 | thumbnailUrl: youtubeDLInfo.thumbnailUrl, | ||
130 | downloadThumbnail, | ||
131 | downloadPreview | ||
132 | } | ||
133 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | ||
134 | |||
135 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) | ||
136 | |||
137 | return res.json(videoImport.toFormattedJSON()).end() | ||
138 | } | 206 | } |
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 338c96582..f4c1c8b07 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts | |||
@@ -17,6 +17,7 @@ import { VideoModel } from '../../models/video/video' | |||
17 | import { exists, isArray, isFileValid } from './misc' | 17 | import { exists, isArray, isFileValid } from './misc' |
18 | import { VideoChannelModel } from '../../models/video/video-channel' | 18 | import { VideoChannelModel } from '../../models/video/video-channel' |
19 | import { UserModel } from '../../models/account/user' | 19 | import { UserModel } from '../../models/account/user' |
20 | import * as magnetUtil from 'magnet-uri' | ||
20 | 21 | ||
21 | const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS | 22 | const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS |
22 | const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES | 23 | const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES |
@@ -126,6 +127,13 @@ function isVideoFileSizeValid (value: string) { | |||
126 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) | 127 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) |
127 | } | 128 | } |
128 | 129 | ||
130 | function isVideoMagnetUriValid (value: string) { | ||
131 | if (!exists(value)) return false | ||
132 | |||
133 | const parsed = magnetUtil.decode(value) | ||
134 | return parsed && isVideoFileInfoHashValid(parsed.infoHash) | ||
135 | } | ||
136 | |||
129 | function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) { | 137 | function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) { |
130 | // Retrieve the user who did the request | 138 | // Retrieve the user who did the request |
131 | if (video.isOwned() === false) { | 139 | if (video.isOwned() === false) { |
@@ -214,6 +222,7 @@ export { | |||
214 | isScheduleVideoUpdatePrivacyValid, | 222 | isScheduleVideoUpdatePrivacyValid, |
215 | isVideoAbuseReasonValid, | 223 | isVideoAbuseReasonValid, |
216 | isVideoFile, | 224 | isVideoFile, |
225 | isVideoMagnetUriValid, | ||
217 | isVideoStateValid, | 226 | isVideoStateValid, |
218 | isVideoViewsValid, | 227 | isVideoViewsValid, |
219 | isVideoRatingTypeValid, | 228 | isVideoRatingTypeValid, |
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 7abcec5d7..f4cc5547d 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts | |||
@@ -9,6 +9,8 @@ import { ApplicationModel } from '../models/application/application' | |||
9 | import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' | 9 | import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' |
10 | import { logger } from './logger' | 10 | import { logger } from './logger' |
11 | import { isArray } from './custom-validators/misc' | 11 | import { isArray } from './custom-validators/misc' |
12 | import * as crypto from "crypto" | ||
13 | import { join } from "path" | ||
12 | 14 | ||
13 | const isCidr = require('is-cidr') | 15 | const isCidr = require('is-cidr') |
14 | 16 | ||
@@ -181,8 +183,14 @@ async function getServerActor () { | |||
181 | return Promise.resolve(serverActor) | 183 | return Promise.resolve(serverActor) |
182 | } | 184 | } |
183 | 185 | ||
186 | function generateVideoTmpPath (id: string) { | ||
187 | const hash = crypto.createHash('sha256').update(id).digest('hex') | ||
188 | return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') | ||
189 | } | ||
190 | |||
184 | type SortType = { sortModel: any, sortValue: string } | 191 | type SortType = { sortModel: any, sortValue: string } |
185 | 192 | ||
193 | |||
186 | // --------------------------------------------------------------------------- | 194 | // --------------------------------------------------------------------------- |
187 | 195 | ||
188 | export { | 196 | export { |
@@ -195,5 +203,6 @@ export { | |||
195 | computeResolutionsToTranscode, | 203 | computeResolutionsToTranscode, |
196 | resetSequelizeInstance, | 204 | resetSequelizeInstance, |
197 | getServerActor, | 205 | getServerActor, |
198 | SortType | 206 | SortType, |
207 | generateVideoTmpPath | ||
199 | } | 208 | } |
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts new file mode 100644 index 000000000..fce88a1f6 --- /dev/null +++ b/server/helpers/webtorrent.ts | |||
@@ -0,0 +1,31 @@ | |||
1 | import { logger } from './logger' | ||
2 | import { generateVideoTmpPath } from './utils' | ||
3 | import * as WebTorrent from 'webtorrent' | ||
4 | import { createWriteStream } from 'fs' | ||
5 | |||
6 | function downloadWebTorrentVideo (target: string) { | ||
7 | const path = generateVideoTmpPath(target) | ||
8 | |||
9 | logger.info('Importing torrent video %s', target) | ||
10 | |||
11 | return new Promise<string>((res, rej) => { | ||
12 | const webtorrent = new WebTorrent() | ||
13 | |||
14 | const torrent = webtorrent.add(target, torrent => { | ||
15 | if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target) | ||
16 | |||
17 | const file = torrent.files[ 0 ] | ||
18 | file.createReadStream().pipe(createWriteStream(path)) | ||
19 | }) | ||
20 | |||
21 | torrent.on('done', () => res(path)) | ||
22 | |||
23 | torrent.on('error', err => rej(err)) | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | // --------------------------------------------------------------------------- | ||
28 | |||
29 | export { | ||
30 | downloadWebTorrentVideo | ||
31 | } | ||
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index c59ab9de0..77986f407 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts | |||
@@ -1,18 +1,17 @@ | |||
1 | import * as youtubeDL from 'youtube-dl' | 1 | import * as youtubeDL from 'youtube-dl' |
2 | import { truncate } from 'lodash' | 2 | import { truncate } from 'lodash' |
3 | import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' | 3 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' |
4 | import { join } from 'path' | ||
5 | import * as crypto from 'crypto' | ||
6 | import { logger } from './logger' | 4 | import { logger } from './logger' |
5 | import { generateVideoTmpPath } from './utils' | ||
7 | 6 | ||
8 | export type YoutubeDLInfo = { | 7 | export type YoutubeDLInfo = { |
9 | name: string | 8 | name?: string |
10 | description: string | 9 | description?: string |
11 | category: number | 10 | category?: number |
12 | licence: number | 11 | licence?: number |
13 | nsfw: boolean | 12 | nsfw?: boolean |
14 | tags: string[] | 13 | tags?: string[] |
15 | thumbnailUrl: string | 14 | thumbnailUrl?: string |
16 | } | 15 | } |
17 | 16 | ||
18 | function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> { | 17 | function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> { |
@@ -30,10 +29,9 @@ function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> { | |||
30 | } | 29 | } |
31 | 30 | ||
32 | function downloadYoutubeDLVideo (url: string) { | 31 | function downloadYoutubeDLVideo (url: string) { |
33 | const hash = crypto.createHash('sha256').update(url).digest('hex') | 32 | const path = generateVideoTmpPath(url) |
34 | const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') | ||
35 | 33 | ||
36 | logger.info('Importing video %s', url) | 34 | logger.info('Importing youtubeDL video %s', url) |
37 | 35 | ||
38 | const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] | 36 | const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] |
39 | 37 | ||
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 74fe7965d..243d544ea 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -15,7 +15,7 @@ let config: IConfig = require('config') | |||
15 | 15 | ||
16 | // --------------------------------------------------------------------------- | 16 | // --------------------------------------------------------------------------- |
17 | 17 | ||
18 | const LAST_MIGRATION_VERSION = 240 | 18 | const LAST_MIGRATION_VERSION = 245 |
19 | 19 | ||
20 | // --------------------------------------------------------------------------- | 20 | // --------------------------------------------------------------------------- |
21 | 21 | ||
@@ -271,7 +271,8 @@ const CONSTRAINTS_FIELDS = { | |||
271 | } | 271 | } |
272 | }, | 272 | }, |
273 | VIDEO_IMPORTS: { | 273 | VIDEO_IMPORTS: { |
274 | URL: { min: 3, max: 2000 } // Length | 274 | URL: { min: 3, max: 2000 }, // Length |
275 | TORRENT_NAME: { min: 3, max: 255 }, // Length | ||
275 | }, | 276 | }, |
276 | VIDEOS: { | 277 | VIDEOS: { |
277 | NAME: { min: 3, max: 120 }, // Length | 278 | NAME: { min: 3, max: 120 }, // Length |
diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts new file mode 100644 index 000000000..87603b006 --- /dev/null +++ b/server/initializers/migrations/0245-import-magnet.ts | |||
@@ -0,0 +1,42 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | import { Migration } from '../../models/migrations' | ||
3 | import { CONSTRAINTS_FIELDS } from '../index' | ||
4 | |||
5 | async function up (utils: { | ||
6 | transaction: Sequelize.Transaction | ||
7 | queryInterface: Sequelize.QueryInterface | ||
8 | sequelize: Sequelize.Sequelize | ||
9 | }): Promise<any> { | ||
10 | { | ||
11 | const data = { | ||
12 | type: Sequelize.STRING, | ||
13 | allowNull: true, | ||
14 | defaultValue: null | ||
15 | } as Migration.String | ||
16 | await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data) | ||
17 | } | ||
18 | |||
19 | { | ||
20 | const data = { | ||
21 | type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max), | ||
22 | allowNull: true, | ||
23 | defaultValue: null | ||
24 | } | ||
25 | await utils.queryInterface.addColumn('videoImport', 'magnetUri', data) | ||
26 | } | ||
27 | |||
28 | { | ||
29 | const data = { | ||
30 | type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max), | ||
31 | allowNull: true, | ||
32 | defaultValue: null | ||
33 | } | ||
34 | await utils.queryInterface.addColumn('videoImport', 'torrentName', data) | ||
35 | } | ||
36 | } | ||
37 | |||
38 | function down (options) { | ||
39 | throw new Error('Not implemented.') | ||
40 | } | ||
41 | |||
42 | export { up, down } | ||
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index cdfe412cc..c457b71fc 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared' | |||
13 | import { JobQueue } from '../index' | 13 | import { JobQueue } from '../index' |
14 | import { federateVideoIfNeeded } from '../../activitypub' | 14 | import { federateVideoIfNeeded } from '../../activitypub' |
15 | import { VideoModel } from '../../../models/video/video' | 15 | import { VideoModel } from '../../../models/video/video' |
16 | import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' | ||
16 | 17 | ||
17 | export type VideoImportPayload = { | 18 | type VideoImportYoutubeDLPayload = { |
18 | type: 'youtube-dl' | 19 | type: 'youtube-dl' |
19 | videoImportId: number | 20 | videoImportId: number |
21 | |||
20 | thumbnailUrl: string | 22 | thumbnailUrl: string |
21 | downloadThumbnail: boolean | 23 | downloadThumbnail: boolean |
22 | downloadPreview: boolean | 24 | downloadPreview: boolean |
23 | } | 25 | } |
24 | 26 | ||
27 | type VideoImportTorrentPayload = { | ||
28 | type: 'magnet-uri' | ||
29 | videoImportId: number | ||
30 | } | ||
31 | |||
32 | export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload | ||
33 | |||
25 | async function processVideoImport (job: Bull.Job) { | 34 | async function processVideoImport (job: Bull.Job) { |
26 | const payload = job.data as VideoImportPayload | 35 | const payload = job.data as VideoImportPayload |
27 | logger.info('Processing video import in job %d.', job.id) | ||
28 | 36 | ||
29 | const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) | 37 | if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload) |
38 | if (payload.type === 'magnet-uri') return processTorrentImport(job, payload) | ||
39 | } | ||
40 | |||
41 | // --------------------------------------------------------------------------- | ||
42 | |||
43 | export { | ||
44 | processVideoImport | ||
45 | } | ||
46 | |||
47 | // --------------------------------------------------------------------------- | ||
48 | |||
49 | async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) { | ||
50 | logger.info('Processing torrent video import in job %d.', job.id) | ||
51 | |||
52 | const videoImport = await getVideoImportOrDie(payload.videoImportId) | ||
53 | const options = { | ||
54 | videoImportId: payload.videoImportId, | ||
55 | |||
56 | downloadThumbnail: false, | ||
57 | downloadPreview: false, | ||
58 | |||
59 | generateThumbnail: true, | ||
60 | generatePreview: true | ||
61 | } | ||
62 | return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options) | ||
63 | } | ||
64 | |||
65 | async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) { | ||
66 | logger.info('Processing youtubeDL video import in job %d.', job.id) | ||
67 | |||
68 | const videoImport = await getVideoImportOrDie(payload.videoImportId) | ||
69 | const options = { | ||
70 | videoImportId: videoImport.id, | ||
71 | |||
72 | downloadThumbnail: payload.downloadThumbnail, | ||
73 | downloadPreview: payload.downloadPreview, | ||
74 | thumbnailUrl: payload.thumbnailUrl, | ||
75 | |||
76 | generateThumbnail: false, | ||
77 | generatePreview: false | ||
78 | } | ||
79 | |||
80 | return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options) | ||
81 | } | ||
82 | |||
83 | async function getVideoImportOrDie (videoImportId: number) { | ||
84 | const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId) | ||
30 | if (!videoImport || !videoImport.Video) { | 85 | if (!videoImport || !videoImport.Video) { |
31 | throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') | 86 | throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') |
32 | } | 87 | } |
33 | 88 | ||
89 | return videoImport | ||
90 | } | ||
91 | |||
92 | type ProcessFileOptions = { | ||
93 | videoImportId: number | ||
94 | |||
95 | downloadThumbnail: boolean | ||
96 | downloadPreview: boolean | ||
97 | thumbnailUrl?: string | ||
98 | |||
99 | generateThumbnail: boolean | ||
100 | generatePreview: boolean | ||
101 | } | ||
102 | async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) { | ||
34 | let tempVideoPath: string | 103 | let tempVideoPath: string |
35 | let videoDestFile: string | 104 | let videoDestFile: string |
36 | let videoFile: VideoFileModel | 105 | let videoFile: VideoFileModel |
37 | try { | 106 | try { |
38 | // Download video from youtubeDL | 107 | // Download video from youtubeDL |
39 | tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) | 108 | tempVideoPath = await downloader() |
40 | 109 | ||
41 | // Get information about this video | 110 | // Get information about this video |
42 | const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) | 111 | const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) |
@@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) { | |||
62 | tempVideoPath = null // This path is not used anymore | 131 | tempVideoPath = null // This path is not used anymore |
63 | 132 | ||
64 | // Process thumbnail | 133 | // Process thumbnail |
65 | if (payload.downloadThumbnail) { | 134 | if (options.downloadThumbnail) { |
66 | if (payload.thumbnailUrl) { | 135 | if (options.thumbnailUrl) { |
67 | const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) | 136 | const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) |
68 | await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) | 137 | await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath) |
69 | } else { | 138 | } else { |
70 | await videoImport.Video.createThumbnail(videoFile) | 139 | await videoImport.Video.createThumbnail(videoFile) |
71 | } | 140 | } |
141 | } else if (options.generateThumbnail) { | ||
142 | await videoImport.Video.createThumbnail(videoFile) | ||
72 | } | 143 | } |
73 | 144 | ||
74 | // Process preview | 145 | // Process preview |
75 | if (payload.downloadPreview) { | 146 | if (options.downloadPreview) { |
76 | if (payload.thumbnailUrl) { | 147 | if (options.thumbnailUrl) { |
77 | const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) | 148 | const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) |
78 | await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) | 149 | await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath) |
79 | } else { | 150 | } else { |
80 | await videoImport.Video.createPreview(videoFile) | 151 | await videoImport.Video.createPreview(videoFile) |
81 | } | 152 | } |
153 | } else if (options.generatePreview) { | ||
154 | await videoImport.Video.createPreview(videoFile) | ||
82 | } | 155 | } |
83 | 156 | ||
84 | // Create torrent | 157 | // Create torrent |
@@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) { | |||
137 | throw err | 210 | throw err |
138 | } | 211 | } |
139 | } | 212 | } |
140 | |||
141 | // --------------------------------------------------------------------------- | ||
142 | |||
143 | export { | ||
144 | processVideoImport | ||
145 | } | ||
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 8a24604e1..ddb357db5 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | |||
32 | 'video-import': processVideoImport | 32 | 'video-import': processVideoImport |
33 | } | 33 | } |
34 | 34 | ||
35 | const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { | ||
36 | 'activitypub-http-broadcast': true, | ||
37 | 'activitypub-http-unicast': true, | ||
38 | 'activitypub-http-fetcher': true, | ||
39 | 'activitypub-follow': true | ||
40 | } | ||
41 | |||
42 | const jobTypes: JobType[] = [ | 35 | const jobTypes: JobType[] = [ |
43 | 'activitypub-follow', | 36 | 'activitypub-follow', |
44 | 'activitypub-http-broadcast', | 37 | 'activitypub-http-broadcast', |
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts index d806edfa3..8ec9373fb 100644 --- a/server/middlewares/validators/video-imports.ts +++ b/server/middlewares/validators/video-imports.ts | |||
@@ -6,14 +6,19 @@ import { areValidationErrors } from './utils' | |||
6 | import { getCommonVideoAttributes } from './videos' | 6 | import { getCommonVideoAttributes } from './videos' |
7 | import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' | 7 | import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' |
8 | import { cleanUpReqFiles } from '../../helpers/utils' | 8 | import { cleanUpReqFiles } from '../../helpers/utils' |
9 | import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' | 9 | import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos' |
10 | import { CONFIG } from '../../initializers/constants' | 10 | import { CONFIG } from '../../initializers/constants' |
11 | 11 | ||
12 | const videoImportAddValidator = getCommonVideoAttributes().concat([ | 12 | const videoImportAddValidator = getCommonVideoAttributes().concat([ |
13 | body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), | ||
14 | body('channelId') | 13 | body('channelId') |
15 | .toInt() | 14 | .toInt() |
16 | .custom(isIdValid).withMessage('Should have correct video channel id'), | 15 | .custom(isIdValid).withMessage('Should have correct video channel id'), |
16 | body('targetUrl') | ||
17 | .optional() | ||
18 | .custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), | ||
19 | body('magnetUri') | ||
20 | .optional() | ||
21 | .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'), | ||
17 | body('name') | 22 | body('name') |
18 | .optional() | 23 | .optional() |
19 | .custom(isVideoNameValid).withMessage('Should have a valid name'), | 24 | .custom(isVideoNameValid).withMessage('Should have a valid name'), |
@@ -34,6 +39,15 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ | |||
34 | 39 | ||
35 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) | 40 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) |
36 | 41 | ||
42 | // Check we have at least 1 required param | ||
43 | if (!req.body.targetUrl && !req.body.magnetUri) { | ||
44 | cleanUpReqFiles(req) | ||
45 | |||
46 | return res.status(400) | ||
47 | .json({ error: 'Should have a magnetUri or a targetUrl.' }) | ||
48 | .end() | ||
49 | } | ||
50 | |||
37 | return next() | 51 | return next() |
38 | } | 52 | } |
39 | ]) | 53 | ]) |
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index eca87163d..55fca28b8 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts | |||
@@ -21,6 +21,7 @@ import { VideoImport, VideoImportState } from '../../../shared' | |||
21 | import { VideoChannelModel } from './video-channel' | 21 | import { VideoChannelModel } from './video-channel' |
22 | import { AccountModel } from '../account/account' | 22 | import { AccountModel } from '../account/account' |
23 | import { TagModel } from './tag' | 23 | import { TagModel } from './tag' |
24 | import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' | ||
24 | 25 | ||
25 | @DefaultScope({ | 26 | @DefaultScope({ |
26 | include: [ | 27 | include: [ |
@@ -62,11 +63,23 @@ export class VideoImportModel extends Model<VideoImportModel> { | |||
62 | @UpdatedAt | 63 | @UpdatedAt |
63 | updatedAt: Date | 64 | updatedAt: Date |
64 | 65 | ||
65 | @AllowNull(false) | 66 | @AllowNull(true) |
67 | @Default(null) | ||
66 | @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) | 68 | @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) |
67 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) | 69 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) |
68 | targetUrl: string | 70 | targetUrl: string |
69 | 71 | ||
72 | @AllowNull(true) | ||
73 | @Default(null) | ||
74 | @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri')) | ||
75 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs | ||
76 | magnetUri: string | ||
77 | |||
78 | @AllowNull(true) | ||
79 | @Default(null) | ||
80 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max)) | ||
81 | torrentName: string | ||
82 | |||
70 | @AllowNull(false) | 83 | @AllowNull(false) |
71 | @Default(null) | 84 | @Default(null) |
72 | @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) | 85 | @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) |