diff options
Diffstat (limited to 'server')
53 files changed, 2267 insertions, 214 deletions
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 9c1b2818c..950a1498e 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts | |||
@@ -9,10 +9,13 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers' | |||
9 | import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' | 9 | import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' |
10 | import { customConfigUpdateValidator } from '../../middlewares/validators/config' | 10 | import { customConfigUpdateValidator } from '../../middlewares/validators/config' |
11 | import { ClientHtml } from '../../lib/client-html' | 11 | import { ClientHtml } from '../../lib/client-html' |
12 | import { CustomConfigAuditView, auditLoggerFactory } from '../../helpers/audit-logger' | ||
12 | 13 | ||
13 | const packageJSON = require('../../../../package.json') | 14 | const packageJSON = require('../../../../package.json') |
14 | const configRouter = express.Router() | 15 | const configRouter = express.Router() |
15 | 16 | ||
17 | const auditLogger = auditLoggerFactory('config') | ||
18 | |||
16 | configRouter.get('/about', getAbout) | 19 | configRouter.get('/about', getAbout) |
17 | configRouter.get('/', | 20 | configRouter.get('/', |
18 | asyncMiddleware(getConfig) | 21 | asyncMiddleware(getConfig) |
@@ -62,6 +65,13 @@ async function getConfig (req: express.Request, res: express.Response, next: exp | |||
62 | transcoding: { | 65 | transcoding: { |
63 | enabledResolutions | 66 | enabledResolutions |
64 | }, | 67 | }, |
68 | import: { | ||
69 | videos: { | ||
70 | http: { | ||
71 | enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED | ||
72 | } | ||
73 | } | ||
74 | }, | ||
65 | avatar: { | 75 | avatar: { |
66 | file: { | 76 | file: { |
67 | size: { | 77 | size: { |
@@ -119,6 +129,11 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex | |||
119 | async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { | 129 | async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { |
120 | await unlinkPromise(CONFIG.CUSTOM_FILE) | 130 | await unlinkPromise(CONFIG.CUSTOM_FILE) |
121 | 131 | ||
132 | auditLogger.delete( | ||
133 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
134 | new CustomConfigAuditView(customConfig()) | ||
135 | ) | ||
136 | |||
122 | reloadConfig() | 137 | reloadConfig() |
123 | ClientHtml.invalidCache() | 138 | ClientHtml.invalidCache() |
124 | 139 | ||
@@ -129,6 +144,7 @@ async function deleteCustomConfig (req: express.Request, res: express.Response, | |||
129 | 144 | ||
130 | async function updateCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { | 145 | async function updateCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { |
131 | const toUpdate: CustomConfig = req.body | 146 | const toUpdate: CustomConfig = req.body |
147 | const oldCustomConfigAuditKeys = new CustomConfigAuditView(customConfig()) | ||
132 | 148 | ||
133 | // Force number conversion | 149 | // Force number conversion |
134 | toUpdate.cache.previews.size = parseInt('' + toUpdate.cache.previews.size, 10) | 150 | toUpdate.cache.previews.size = parseInt('' + toUpdate.cache.previews.size, 10) |
@@ -150,6 +166,13 @@ async function updateCustomConfig (req: express.Request, res: express.Response, | |||
150 | ClientHtml.invalidCache() | 166 | ClientHtml.invalidCache() |
151 | 167 | ||
152 | const data = customConfig() | 168 | const data = customConfig() |
169 | |||
170 | auditLogger.update( | ||
171 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
172 | new CustomConfigAuditView(data), | ||
173 | oldCustomConfigAuditKeys | ||
174 | ) | ||
175 | |||
153 | return res.json(data).end() | 176 | return res.json(data).end() |
154 | } | 177 | } |
155 | 178 | ||
@@ -209,6 +232,13 @@ function customConfig (): CustomConfig { | |||
209 | '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], | 232 | '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], |
210 | '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] | 233 | '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] |
211 | } | 234 | } |
235 | }, | ||
236 | import: { | ||
237 | videos: { | ||
238 | http: { | ||
239 | enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED | ||
240 | } | ||
241 | } | ||
212 | } | 242 | } |
213 | } | 243 | } |
214 | } | 244 | } |
diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts index c80f27a23..879ba3f91 100644 --- a/server/controllers/api/users.ts +++ b/server/controllers/api/users.ts | |||
@@ -29,7 +29,12 @@ import { | |||
29 | usersUpdateValidator, | 29 | usersUpdateValidator, |
30 | usersVideoRatingValidator | 30 | usersVideoRatingValidator |
31 | } from '../../middlewares' | 31 | } from '../../middlewares' |
32 | import { usersAskResetPasswordValidator, usersResetPasswordValidator, videosSortValidator } from '../../middlewares/validators' | 32 | import { |
33 | usersAskResetPasswordValidator, | ||
34 | usersResetPasswordValidator, | ||
35 | videoImportsSortValidator, | ||
36 | videosSortValidator | ||
37 | } from '../../middlewares/validators' | ||
33 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | 38 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' |
34 | import { UserModel } from '../../models/account/user' | 39 | import { UserModel } from '../../models/account/user' |
35 | import { OAuthTokenModel } from '../../models/oauth/oauth-token' | 40 | import { OAuthTokenModel } from '../../models/oauth/oauth-token' |
@@ -39,6 +44,10 @@ import { createReqFiles } from '../../helpers/express-utils' | |||
39 | import { UserVideoQuota } from '../../../shared/models/users/user-video-quota.model' | 44 | import { UserVideoQuota } from '../../../shared/models/users/user-video-quota.model' |
40 | import { updateAvatarValidator } from '../../middlewares/validators/avatar' | 45 | import { updateAvatarValidator } from '../../middlewares/validators/avatar' |
41 | import { updateActorAvatarFile } from '../../lib/avatar' | 46 | import { updateActorAvatarFile } from '../../lib/avatar' |
47 | import { auditLoggerFactory, UserAuditView } from '../../helpers/audit-logger' | ||
48 | import { VideoImportModel } from '../../models/video/video-import' | ||
49 | |||
50 | const auditLogger = auditLoggerFactory('users') | ||
42 | 51 | ||
43 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) | 52 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) |
44 | const loginRateLimiter = new RateLimit({ | 53 | const loginRateLimiter = new RateLimit({ |
@@ -59,6 +68,15 @@ usersRouter.get('/me/video-quota-used', | |||
59 | asyncMiddleware(getUserVideoQuotaUsed) | 68 | asyncMiddleware(getUserVideoQuotaUsed) |
60 | ) | 69 | ) |
61 | 70 | ||
71 | usersRouter.get('/me/videos/imports', | ||
72 | authenticate, | ||
73 | paginationValidator, | ||
74 | videoImportsSortValidator, | ||
75 | setDefaultSort, | ||
76 | setDefaultPagination, | ||
77 | asyncMiddleware(getUserVideoImports) | ||
78 | ) | ||
79 | |||
62 | usersRouter.get('/me/videos', | 80 | usersRouter.get('/me/videos', |
63 | authenticate, | 81 | authenticate, |
64 | paginationValidator, | 82 | paginationValidator, |
@@ -175,6 +193,18 @@ async function getUserVideos (req: express.Request, res: express.Response, next: | |||
175 | return res.json(getFormattedObjects(resultList.data, resultList.total, { additionalAttributes })) | 193 | return res.json(getFormattedObjects(resultList.data, resultList.total, { additionalAttributes })) |
176 | } | 194 | } |
177 | 195 | ||
196 | async function getUserVideoImports (req: express.Request, res: express.Response, next: express.NextFunction) { | ||
197 | const user = res.locals.oauth.token.User as UserModel | ||
198 | const resultList = await VideoImportModel.listUserVideoImportsForApi( | ||
199 | user.Account.id, | ||
200 | req.query.start as number, | ||
201 | req.query.count as number, | ||
202 | req.query.sort | ||
203 | ) | ||
204 | |||
205 | return res.json(getFormattedObjects(resultList.data, resultList.total)) | ||
206 | } | ||
207 | |||
178 | async function createUser (req: express.Request, res: express.Response) { | 208 | async function createUser (req: express.Request, res: express.Response) { |
179 | const body: UserCreate = req.body | 209 | const body: UserCreate = req.body |
180 | const userToCreate = new UserModel({ | 210 | const userToCreate = new UserModel({ |
@@ -189,6 +219,7 @@ async function createUser (req: express.Request, res: express.Response) { | |||
189 | 219 | ||
190 | const { user, account } = await createUserAccountAndChannel(userToCreate) | 220 | const { user, account } = await createUserAccountAndChannel(userToCreate) |
191 | 221 | ||
222 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) | ||
192 | logger.info('User %s with its channel and account created.', body.username) | 223 | logger.info('User %s with its channel and account created.', body.username) |
193 | 224 | ||
194 | return res.json({ | 225 | return res.json({ |
@@ -205,7 +236,7 @@ async function createUser (req: express.Request, res: express.Response) { | |||
205 | async function registerUser (req: express.Request, res: express.Response) { | 236 | async function registerUser (req: express.Request, res: express.Response) { |
206 | const body: UserCreate = req.body | 237 | const body: UserCreate = req.body |
207 | 238 | ||
208 | const user = new UserModel({ | 239 | const userToCreate = new UserModel({ |
209 | username: body.username, | 240 | username: body.username, |
210 | password: body.password, | 241 | password: body.password, |
211 | email: body.email, | 242 | email: body.email, |
@@ -215,8 +246,9 @@ async function registerUser (req: express.Request, res: express.Response) { | |||
215 | videoQuota: CONFIG.USER.VIDEO_QUOTA | 246 | videoQuota: CONFIG.USER.VIDEO_QUOTA |
216 | }) | 247 | }) |
217 | 248 | ||
218 | await createUserAccountAndChannel(user) | 249 | const { user } = await createUserAccountAndChannel(userToCreate) |
219 | 250 | ||
251 | auditLogger.create(body.username, new UserAuditView(user.toFormattedJSON())) | ||
220 | logger.info('User %s with its channel and account registered.', body.username) | 252 | logger.info('User %s with its channel and account registered.', body.username) |
221 | 253 | ||
222 | return res.type('json').status(204).end() | 254 | return res.type('json').status(204).end() |
@@ -269,6 +301,8 @@ async function removeUser (req: express.Request, res: express.Response, next: ex | |||
269 | 301 | ||
270 | await user.destroy() | 302 | await user.destroy() |
271 | 303 | ||
304 | auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON())) | ||
305 | |||
272 | return res.sendStatus(204) | 306 | return res.sendStatus(204) |
273 | } | 307 | } |
274 | 308 | ||
@@ -276,6 +310,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr | |||
276 | const body: UserUpdateMe = req.body | 310 | const body: UserUpdateMe = req.body |
277 | 311 | ||
278 | const user: UserModel = res.locals.oauth.token.user | 312 | const user: UserModel = res.locals.oauth.token.user |
313 | const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) | ||
279 | 314 | ||
280 | if (body.password !== undefined) user.password = body.password | 315 | if (body.password !== undefined) user.password = body.password |
281 | if (body.email !== undefined) user.email = body.email | 316 | if (body.email !== undefined) user.email = body.email |
@@ -290,6 +325,12 @@ async function updateMe (req: express.Request, res: express.Response, next: expr | |||
290 | await user.Account.save({ transaction: t }) | 325 | await user.Account.save({ transaction: t }) |
291 | 326 | ||
292 | await sendUpdateActor(user.Account, t) | 327 | await sendUpdateActor(user.Account, t) |
328 | |||
329 | auditLogger.update( | ||
330 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
331 | new UserAuditView(user.toFormattedJSON()), | ||
332 | oldUserAuditView | ||
333 | ) | ||
293 | }) | 334 | }) |
294 | 335 | ||
295 | return res.sendStatus(204) | 336 | return res.sendStatus(204) |
@@ -297,10 +338,18 @@ async function updateMe (req: express.Request, res: express.Response, next: expr | |||
297 | 338 | ||
298 | async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { | 339 | async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { |
299 | const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] | 340 | const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] |
300 | const account = res.locals.oauth.token.user.Account | 341 | const user: UserModel = res.locals.oauth.token.user |
342 | const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) | ||
343 | const account = user.Account | ||
301 | 344 | ||
302 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) | 345 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) |
303 | 346 | ||
347 | auditLogger.update( | ||
348 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
349 | new UserAuditView(user.toFormattedJSON()), | ||
350 | oldUserAuditView | ||
351 | ) | ||
352 | |||
304 | return res | 353 | return res |
305 | .json({ | 354 | .json({ |
306 | avatar: avatar.toFormattedJSON() | 355 | avatar: avatar.toFormattedJSON() |
@@ -310,20 +359,27 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next | |||
310 | 359 | ||
311 | async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) { | 360 | async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) { |
312 | const body: UserUpdate = req.body | 361 | const body: UserUpdate = req.body |
313 | const user = res.locals.user as UserModel | 362 | const userToUpdate = res.locals.user as UserModel |
314 | const roleChanged = body.role !== undefined && body.role !== user.role | 363 | const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) |
364 | const roleChanged = body.role !== undefined && body.role !== userToUpdate.role | ||
315 | 365 | ||
316 | if (body.email !== undefined) user.email = body.email | 366 | if (body.email !== undefined) userToUpdate.email = body.email |
317 | if (body.videoQuota !== undefined) user.videoQuota = body.videoQuota | 367 | if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota |
318 | if (body.role !== undefined) user.role = body.role | 368 | if (body.role !== undefined) userToUpdate.role = body.role |
319 | 369 | ||
320 | await user.save() | 370 | const user = await userToUpdate.save() |
321 | 371 | ||
322 | // Destroy user token to refresh rights | 372 | // Destroy user token to refresh rights |
323 | if (roleChanged) { | 373 | if (roleChanged) { |
324 | await OAuthTokenModel.deleteUserToken(user.id) | 374 | await OAuthTokenModel.deleteUserToken(userToUpdate.id) |
325 | } | 375 | } |
326 | 376 | ||
377 | auditLogger.update( | ||
378 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
379 | new UserAuditView(user.toFormattedJSON()), | ||
380 | oldUserAuditView | ||
381 | ) | ||
382 | |||
327 | // Don't need to send this update to followers, these attributes are not propagated | 383 | // Don't need to send this update to followers, these attributes are not propagated |
328 | 384 | ||
329 | return res.sendStatus(204) | 385 | return res.sendStatus(204) |
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 0488ba8f5..3a444547b 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts | |||
@@ -27,7 +27,9 @@ import { logger } from '../../helpers/logger' | |||
27 | import { VideoModel } from '../../models/video/video' | 27 | import { VideoModel } from '../../models/video/video' |
28 | import { updateAvatarValidator } from '../../middlewares/validators/avatar' | 28 | import { updateAvatarValidator } from '../../middlewares/validators/avatar' |
29 | import { updateActorAvatarFile } from '../../lib/avatar' | 29 | import { updateActorAvatarFile } from '../../lib/avatar' |
30 | import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger' | ||
30 | 31 | ||
32 | const auditLogger = auditLoggerFactory('channels') | ||
31 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) | 33 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) |
32 | 34 | ||
33 | const videoChannelRouter = express.Router() | 35 | const videoChannelRouter = express.Router() |
@@ -99,10 +101,17 @@ async function listVideoChannels (req: express.Request, res: express.Response, n | |||
99 | 101 | ||
100 | async function updateVideoChannelAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { | 102 | async function updateVideoChannelAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { |
101 | const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] | 103 | const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] |
102 | const videoChannel = res.locals.videoChannel | 104 | const videoChannel = res.locals.videoChannel as VideoChannelModel |
105 | const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) | ||
103 | 106 | ||
104 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) | 107 | const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) |
105 | 108 | ||
109 | auditLogger.update( | ||
110 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
111 | new VideoChannelAuditView(videoChannel.toFormattedJSON()), | ||
112 | oldVideoChannelAuditKeys | ||
113 | ) | ||
114 | |||
106 | return res | 115 | return res |
107 | .json({ | 116 | .json({ |
108 | avatar: avatar.toFormattedJSON() | 117 | avatar: avatar.toFormattedJSON() |
@@ -121,6 +130,10 @@ async function addVideoChannel (req: express.Request, res: express.Response) { | |||
121 | setAsyncActorKeys(videoChannelCreated.Actor) | 130 | setAsyncActorKeys(videoChannelCreated.Actor) |
122 | .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) | 131 | .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) |
123 | 132 | ||
133 | auditLogger.create( | ||
134 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
135 | new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()) | ||
136 | ) | ||
124 | logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) | 137 | logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) |
125 | 138 | ||
126 | return res.json({ | 139 | return res.json({ |
@@ -134,6 +147,7 @@ async function addVideoChannel (req: express.Request, res: express.Response) { | |||
134 | async function updateVideoChannel (req: express.Request, res: express.Response) { | 147 | async function updateVideoChannel (req: express.Request, res: express.Response) { |
135 | const videoChannelInstance = res.locals.videoChannel as VideoChannelModel | 148 | const videoChannelInstance = res.locals.videoChannel as VideoChannelModel |
136 | const videoChannelFieldsSave = videoChannelInstance.toJSON() | 149 | const videoChannelFieldsSave = videoChannelInstance.toJSON() |
150 | const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) | ||
137 | const videoChannelInfoToUpdate = req.body as VideoChannelUpdate | 151 | const videoChannelInfoToUpdate = req.body as VideoChannelUpdate |
138 | 152 | ||
139 | try { | 153 | try { |
@@ -148,9 +162,14 @@ async function updateVideoChannel (req: express.Request, res: express.Response) | |||
148 | 162 | ||
149 | const videoChannelInstanceUpdated = await videoChannelInstance.save(sequelizeOptions) | 163 | const videoChannelInstanceUpdated = await videoChannelInstance.save(sequelizeOptions) |
150 | await sendUpdateActor(videoChannelInstanceUpdated, t) | 164 | await sendUpdateActor(videoChannelInstanceUpdated, t) |
151 | }) | ||
152 | 165 | ||
153 | logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) | 166 | auditLogger.update( |
167 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
168 | new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), | ||
169 | oldVideoChannelAuditKeys | ||
170 | ) | ||
171 | logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) | ||
172 | }) | ||
154 | } catch (err) { | 173 | } catch (err) { |
155 | logger.debug('Cannot update the video channel.', { err }) | 174 | logger.debug('Cannot update the video channel.', { err }) |
156 | 175 | ||
@@ -171,6 +190,10 @@ async function removeVideoChannel (req: express.Request, res: express.Response) | |||
171 | await sequelizeTypescript.transaction(async t => { | 190 | await sequelizeTypescript.transaction(async t => { |
172 | await videoChannelInstance.destroy({ transaction: t }) | 191 | await videoChannelInstance.destroy({ transaction: t }) |
173 | 192 | ||
193 | auditLogger.delete( | ||
194 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
195 | new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) | ||
196 | ) | ||
174 | logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) | 197 | logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) |
175 | }) | 198 | }) |
176 | 199 | ||
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 3413ae894..7782fc639 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts | |||
@@ -18,7 +18,9 @@ import { | |||
18 | import { AccountModel } from '../../../models/account/account' | 18 | import { AccountModel } from '../../../models/account/account' |
19 | import { VideoModel } from '../../../models/video/video' | 19 | import { VideoModel } from '../../../models/video/video' |
20 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | 20 | import { VideoAbuseModel } from '../../../models/video/video-abuse' |
21 | import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger' | ||
21 | 22 | ||
23 | const auditLogger = auditLoggerFactory('abuse') | ||
22 | const abuseVideoRouter = express.Router() | 24 | const abuseVideoRouter = express.Router() |
23 | 25 | ||
24 | abuseVideoRouter.get('/abuse', | 26 | abuseVideoRouter.get('/abuse', |
@@ -64,14 +66,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { | |||
64 | await sequelizeTypescript.transaction(async t => { | 66 | await sequelizeTypescript.transaction(async t => { |
65 | const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) | 67 | const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) |
66 | videoAbuseInstance.Video = videoInstance | 68 | videoAbuseInstance.Video = videoInstance |
69 | videoAbuseInstance.Account = reporterAccount | ||
67 | 70 | ||
68 | // We send the video abuse to the origin server | 71 | // We send the video abuse to the origin server |
69 | if (videoInstance.isOwned() === false) { | 72 | if (videoInstance.isOwned() === false) { |
70 | await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) | 73 | await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) |
71 | } | 74 | } |
72 | }) | ||
73 | 75 | ||
74 | logger.info('Abuse report for video %s created.', videoInstance.name) | 76 | auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON())) |
77 | logger.info('Abuse report for video %s created.', videoInstance.name) | ||
78 | }) | ||
75 | 79 | ||
76 | return res.type('json').status(204).end() | 80 | return res.type('json').status(204).end() |
77 | } | 81 | } |
diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts index bbeb0d557..e35247829 100644 --- a/server/controllers/api/videos/comment.ts +++ b/server/controllers/api/videos/comment.ts | |||
@@ -23,7 +23,9 @@ import { | |||
23 | } from '../../../middlewares/validators/video-comments' | 23 | } from '../../../middlewares/validators/video-comments' |
24 | import { VideoModel } from '../../../models/video/video' | 24 | import { VideoModel } from '../../../models/video/video' |
25 | import { VideoCommentModel } from '../../../models/video/video-comment' | 25 | import { VideoCommentModel } from '../../../models/video/video-comment' |
26 | import { auditLoggerFactory, CommentAuditView } from '../../../helpers/audit-logger' | ||
26 | 27 | ||
28 | const auditLogger = auditLoggerFactory('comments') | ||
27 | const videoCommentRouter = express.Router() | 29 | const videoCommentRouter = express.Router() |
28 | 30 | ||
29 | videoCommentRouter.get('/:videoId/comment-threads', | 31 | videoCommentRouter.get('/:videoId/comment-threads', |
@@ -107,6 +109,8 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons | |||
107 | }, t) | 109 | }, t) |
108 | }) | 110 | }) |
109 | 111 | ||
112 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) | ||
113 | |||
110 | return res.json({ | 114 | return res.json({ |
111 | comment: comment.toFormattedJSON() | 115 | comment: comment.toFormattedJSON() |
112 | }).end() | 116 | }).end() |
@@ -124,6 +128,8 @@ async function addVideoCommentReply (req: express.Request, res: express.Response | |||
124 | }, t) | 128 | }, t) |
125 | }) | 129 | }) |
126 | 130 | ||
131 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON())) | ||
132 | |||
127 | return res.json({ | 133 | return res.json({ |
128 | comment: comment.toFormattedJSON() | 134 | comment: comment.toFormattedJSON() |
129 | }).end() | 135 | }).end() |
@@ -136,6 +142,10 @@ async function removeVideoComment (req: express.Request, res: express.Response) | |||
136 | await videoCommentInstance.destroy({ transaction: t }) | 142 | await videoCommentInstance.destroy({ transaction: t }) |
137 | }) | 143 | }) |
138 | 144 | ||
145 | auditLogger.delete( | ||
146 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
147 | new CommentAuditView(videoCommentInstance.toFormattedJSON()) | ||
148 | ) | ||
139 | logger.info('Video comment %d deleted.', videoCommentInstance.id) | 149 | logger.info('Video comment %d deleted.', videoCommentInstance.id) |
140 | 150 | ||
141 | return res.type('json').status(204).end() | 151 | return res.type('json').status(204).end() |
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts new file mode 100644 index 000000000..30a7d816c --- /dev/null +++ b/server/controllers/api/videos/import.ts | |||
@@ -0,0 +1,138 @@ | |||
1 | import * as express from 'express' | ||
2 | import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger' | ||
3 | import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares' | ||
4 | import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers' | ||
5 | import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl' | ||
6 | import { createReqFiles } from '../../../helpers/express-utils' | ||
7 | import { logger } from '../../../helpers/logger' | ||
8 | import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared' | ||
9 | import { VideoModel } from '../../../models/video/video' | ||
10 | import { getVideoActivityPubUrl } from '../../../lib/activitypub' | ||
11 | import { TagModel } from '../../../models/video/tag' | ||
12 | import { VideoImportModel } from '../../../models/video/video-import' | ||
13 | import { JobQueue } from '../../../lib/job-queue/job-queue' | ||
14 | import { processImage } from '../../../helpers/image-utils' | ||
15 | import { join } from 'path' | ||
16 | |||
17 | const auditLogger = auditLoggerFactory('video-imports') | ||
18 | const videoImportsRouter = express.Router() | ||
19 | |||
20 | const reqVideoFileImport = createReqFiles( | ||
21 | [ 'thumbnailfile', 'previewfile' ], | ||
22 | IMAGE_MIMETYPE_EXT, | ||
23 | { | ||
24 | thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR, | ||
25 | previewfile: CONFIG.STORAGE.PREVIEWS_DIR | ||
26 | } | ||
27 | ) | ||
28 | |||
29 | videoImportsRouter.post('/imports', | ||
30 | authenticate, | ||
31 | reqVideoFileImport, | ||
32 | asyncMiddleware(videoImportAddValidator), | ||
33 | asyncRetryTransactionMiddleware(addVideoImport) | ||
34 | ) | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | videoImportsRouter | ||
40 | } | ||
41 | |||
42 | // --------------------------------------------------------------------------- | ||
43 | |||
44 | async function addVideoImport (req: express.Request, res: express.Response) { | ||
45 | const body: VideoImportCreate = req.body | ||
46 | const targetUrl = body.targetUrl | ||
47 | |||
48 | let youtubeDLInfo: YoutubeDLInfo | ||
49 | try { | ||
50 | youtubeDLInfo = await getYoutubeDLInfo(targetUrl) | ||
51 | } catch (err) { | ||
52 | logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err }) | ||
53 | |||
54 | return res.status(400).json({ | ||
55 | error: 'Cannot fetch remote information of this URL.' | ||
56 | }).end() | ||
57 | } | ||
58 | |||
59 | // Create video DB object | ||
60 | const videoData = { | ||
61 | name: body.name || youtubeDLInfo.name, | ||
62 | remote: false, | ||
63 | category: body.category || youtubeDLInfo.category, | ||
64 | licence: body.licence || youtubeDLInfo.licence, | ||
65 | language: body.language || undefined, | ||
66 | commentsEnabled: body.commentsEnabled || true, | ||
67 | waitTranscoding: body.waitTranscoding || false, | ||
68 | state: VideoState.TO_IMPORT, | ||
69 | nsfw: body.nsfw || youtubeDLInfo.nsfw || false, | ||
70 | description: body.description || youtubeDLInfo.description, | ||
71 | support: body.support || null, | ||
72 | privacy: body.privacy || VideoPrivacy.PRIVATE, | ||
73 | duration: 0, // duration will be set by the import job | ||
74 | channelId: res.locals.videoChannel.id | ||
75 | } | ||
76 | const video = new VideoModel(videoData) | ||
77 | video.url = getVideoActivityPubUrl(video) | ||
78 | |||
79 | // Process thumbnail file? | ||
80 | const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined | ||
81 | let downloadThumbnail = true | ||
82 | if (thumbnailField) { | ||
83 | const thumbnailPhysicalFile = thumbnailField[ 0 ] | ||
84 | await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE) | ||
85 | downloadThumbnail = false | ||
86 | } | ||
87 | |||
88 | // Process preview file? | ||
89 | const previewField = req.files ? req.files['previewfile'] : undefined | ||
90 | let downloadPreview = true | ||
91 | if (previewField) { | ||
92 | const previewPhysicalFile = previewField[0] | ||
93 | await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE) | ||
94 | downloadPreview = false | ||
95 | } | ||
96 | |||
97 | const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => { | ||
98 | const sequelizeOptions = { transaction: t } | ||
99 | |||
100 | // Save video object in database | ||
101 | const videoCreated = await video.save(sequelizeOptions) | ||
102 | videoCreated.VideoChannel = res.locals.videoChannel | ||
103 | |||
104 | // Set tags to the video | ||
105 | const tags = body.tags ? body.tags : youtubeDLInfo.tags | ||
106 | if (tags !== undefined) { | ||
107 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
108 | |||
109 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
110 | videoCreated.Tags = tagInstances | ||
111 | } | ||
112 | |||
113 | // Create video import object in database | ||
114 | const videoImport = await VideoImportModel.create({ | ||
115 | targetUrl, | ||
116 | state: VideoImportState.PENDING, | ||
117 | videoId: videoCreated.id | ||
118 | }, sequelizeOptions) | ||
119 | |||
120 | videoImport.Video = videoCreated | ||
121 | |||
122 | return videoImport | ||
123 | }) | ||
124 | |||
125 | // Create job to import the video | ||
126 | const payload = { | ||
127 | type: 'youtube-dl' as 'youtube-dl', | ||
128 | videoImportId: videoImport.id, | ||
129 | thumbnailUrl: youtubeDLInfo.thumbnailUrl, | ||
130 | downloadThumbnail, | ||
131 | downloadPreview | ||
132 | } | ||
133 | await JobQueue.Instance.createJob({ type: 'video-import', payload }) | ||
134 | |||
135 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON())) | ||
136 | |||
137 | return res.json(videoImport.toFormattedJSON()).end() | ||
138 | } | ||
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 101183eab..c9365da08 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -5,6 +5,7 @@ import { renamePromise } from '../../../helpers/core-utils' | |||
5 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' | 5 | import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' |
6 | import { processImage } from '../../../helpers/image-utils' | 6 | import { processImage } from '../../../helpers/image-utils' |
7 | import { logger } from '../../../helpers/logger' | 7 | import { logger } from '../../../helpers/logger' |
8 | import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger' | ||
8 | import { getFormattedObjects, getServerActor, resetSequelizeInstance } from '../../../helpers/utils' | 9 | import { getFormattedObjects, getServerActor, resetSequelizeInstance } from '../../../helpers/utils' |
9 | import { | 10 | import { |
10 | CONFIG, | 11 | CONFIG, |
@@ -53,7 +54,9 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type' | |||
53 | import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils' | 54 | import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils' |
54 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' | 55 | import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' |
55 | import { videoCaptionsRouter } from './captions' | 56 | import { videoCaptionsRouter } from './captions' |
57 | import { videoImportsRouter } from './import' | ||
56 | 58 | ||
59 | const auditLogger = auditLoggerFactory('videos') | ||
57 | const videosRouter = express.Router() | 60 | const videosRouter = express.Router() |
58 | 61 | ||
59 | const reqVideoFileAdd = createReqFiles( | 62 | const reqVideoFileAdd = createReqFiles( |
@@ -79,6 +82,7 @@ videosRouter.use('/', blacklistRouter) | |||
79 | videosRouter.use('/', rateVideoRouter) | 82 | videosRouter.use('/', rateVideoRouter) |
80 | videosRouter.use('/', videoCommentRouter) | 83 | videosRouter.use('/', videoCommentRouter) |
81 | videosRouter.use('/', videoCaptionsRouter) | 84 | videosRouter.use('/', videoCaptionsRouter) |
85 | videosRouter.use('/', videoImportsRouter) | ||
82 | 86 | ||
83 | videosRouter.get('/categories', listVideoCategories) | 87 | videosRouter.get('/categories', listVideoCategories) |
84 | videosRouter.get('/licences', listVideoLicences) | 88 | videosRouter.get('/licences', listVideoLicences) |
@@ -158,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) { | |||
158 | const videoData = { | 162 | const videoData = { |
159 | name: videoInfo.name, | 163 | name: videoInfo.name, |
160 | remote: false, | 164 | remote: false, |
161 | extname: extname(videoPhysicalFile.filename), | ||
162 | category: videoInfo.category, | 165 | category: videoInfo.category, |
163 | licence: videoInfo.licence, | 166 | licence: videoInfo.licence, |
164 | language: videoInfo.language, | 167 | language: videoInfo.language, |
@@ -247,6 +250,7 @@ async function addVideo (req: express.Request, res: express.Response) { | |||
247 | 250 | ||
248 | await federateVideoIfNeeded(video, true, t) | 251 | await federateVideoIfNeeded(video, true, t) |
249 | 252 | ||
253 | auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) | ||
250 | logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) | 254 | logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) |
251 | 255 | ||
252 | return videoCreated | 256 | return videoCreated |
@@ -273,6 +277,7 @@ async function addVideo (req: express.Request, res: express.Response) { | |||
273 | async function updateVideo (req: express.Request, res: express.Response) { | 277 | async function updateVideo (req: express.Request, res: express.Response) { |
274 | const videoInstance: VideoModel = res.locals.video | 278 | const videoInstance: VideoModel = res.locals.video |
275 | const videoFieldsSave = videoInstance.toJSON() | 279 | const videoFieldsSave = videoInstance.toJSON() |
280 | const oldVideoAuditView = new VideoAuditView(videoInstance.toFormattedDetailsJSON()) | ||
276 | const videoInfoToUpdate: VideoUpdate = req.body | 281 | const videoInfoToUpdate: VideoUpdate = req.body |
277 | const wasPrivateVideo = videoInstance.privacy === VideoPrivacy.PRIVATE | 282 | const wasPrivateVideo = videoInstance.privacy === VideoPrivacy.PRIVATE |
278 | 283 | ||
@@ -344,9 +349,14 @@ async function updateVideo (req: express.Request, res: express.Response) { | |||
344 | 349 | ||
345 | const isNewVideo = wasPrivateVideo && videoInstanceUpdated.privacy !== VideoPrivacy.PRIVATE | 350 | const isNewVideo = wasPrivateVideo && videoInstanceUpdated.privacy !== VideoPrivacy.PRIVATE |
346 | await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) | 351 | await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) |
347 | }) | ||
348 | 352 | ||
349 | logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid) | 353 | auditLogger.update( |
354 | res.locals.oauth.token.User.Account.Actor.getIdentifier(), | ||
355 | new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), | ||
356 | oldVideoAuditView | ||
357 | ) | ||
358 | logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid) | ||
359 | }) | ||
350 | } catch (err) { | 360 | } catch (err) { |
351 | // Force fields we want to update | 361 | // Force fields we want to update |
352 | // If the transaction is retried, sequelize will think the object has not changed | 362 | // If the transaction is retried, sequelize will think the object has not changed |
@@ -423,6 +433,7 @@ async function removeVideo (req: express.Request, res: express.Response) { | |||
423 | await videoInstance.destroy({ transaction: t }) | 433 | await videoInstance.destroy({ transaction: t }) |
424 | }) | 434 | }) |
425 | 435 | ||
436 | auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) | ||
426 | logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) | 437 | logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) |
427 | 438 | ||
428 | return res.type('json').status(204).end() | 439 | return res.type('json').status(204).end() |
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index d710f5c97..a9de11fb0 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -24,6 +24,7 @@ function activityPubContextify <T> (data: T) { | |||
24 | views: 'http://schema.org/Number', | 24 | views: 'http://schema.org/Number', |
25 | stats: 'http://schema.org/Number', | 25 | stats: 'http://schema.org/Number', |
26 | size: 'http://schema.org/Number', | 26 | size: 'http://schema.org/Number', |
27 | fps: 'http://schema.org/Number', | ||
27 | commentsEnabled: 'http://schema.org/Boolean', | 28 | commentsEnabled: 'http://schema.org/Boolean', |
28 | waitTranscoding: 'http://schema.org/Boolean', | 29 | waitTranscoding: 'http://schema.org/Boolean', |
29 | support: 'http://schema.org/Text' | 30 | support: 'http://schema.org/Text' |
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts new file mode 100644 index 000000000..db20df20f --- /dev/null +++ b/server/helpers/audit-logger.ts | |||
@@ -0,0 +1,265 @@ | |||
1 | import * as path from 'path' | ||
2 | import { diff } from 'deep-object-diff' | ||
3 | import { chain } from 'lodash' | ||
4 | import * as flatten from 'flat' | ||
5 | import * as winston from 'winston' | ||
6 | import { CONFIG } from '../initializers' | ||
7 | import { jsonLoggerFormat, labelFormatter } from './logger' | ||
8 | import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared' | ||
9 | import { VideoComment } from '../../shared/models/videos/video-comment.model' | ||
10 | import { CustomConfig } from '../../shared/models/server/custom-config.model' | ||
11 | |||
12 | enum AUDIT_TYPE { | ||
13 | CREATE = 'create', | ||
14 | UPDATE = 'update', | ||
15 | DELETE = 'delete' | ||
16 | } | ||
17 | |||
18 | const colors = winston.config.npm.colors | ||
19 | colors.audit = winston.config.npm.colors.info | ||
20 | |||
21 | winston.addColors(colors) | ||
22 | |||
23 | const auditLogger = winston.createLogger({ | ||
24 | levels: { audit: 0 }, | ||
25 | transports: [ | ||
26 | new winston.transports.File({ | ||
27 | filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube-audit.log'), | ||
28 | level: 'audit', | ||
29 | maxsize: 5242880, | ||
30 | maxFiles: 5, | ||
31 | format: winston.format.combine( | ||
32 | winston.format.timestamp(), | ||
33 | labelFormatter, | ||
34 | winston.format.splat(), | ||
35 | jsonLoggerFormat | ||
36 | ) | ||
37 | }) | ||
38 | ], | ||
39 | exitOnError: true | ||
40 | }) | ||
41 | |||
42 | function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) { | ||
43 | let entityInfos: object | ||
44 | if (action === AUDIT_TYPE.UPDATE && oldEntity) { | ||
45 | const oldEntityKeys = oldEntity.toLogKeys() | ||
46 | const diffObject = diff(oldEntityKeys, entity.toLogKeys()) | ||
47 | const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => { | ||
48 | newKeys[`new-${entry[0]}`] = entry[1] | ||
49 | return newKeys | ||
50 | }, {}) | ||
51 | entityInfos = { ...oldEntityKeys, ...diffKeys } | ||
52 | } else { | ||
53 | entityInfos = { ...entity.toLogKeys() } | ||
54 | } | ||
55 | auditLogger.log('audit', JSON.stringify({ | ||
56 | user, | ||
57 | domain, | ||
58 | action, | ||
59 | ...entityInfos | ||
60 | })) | ||
61 | } | ||
62 | |||
63 | function auditLoggerFactory (domain: string) { | ||
64 | return { | ||
65 | create (user: string, entity: EntityAuditView) { | ||
66 | auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity) | ||
67 | }, | ||
68 | update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) { | ||
69 | auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity) | ||
70 | }, | ||
71 | delete (user: string, entity: EntityAuditView) { | ||
72 | auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity) | ||
73 | } | ||
74 | } | ||
75 | } | ||
76 | |||
77 | abstract class EntityAuditView { | ||
78 | constructor (private keysToKeep: Array<string>, private prefix: string, private entityInfos: object) { } | ||
79 | toLogKeys (): object { | ||
80 | return chain(flatten(this.entityInfos, { delimiter: '-', safe: true })) | ||
81 | .pick(this.keysToKeep) | ||
82 | .mapKeys((value, key) => `${this.prefix}-${key}`) | ||
83 | .value() | ||
84 | } | ||
85 | } | ||
86 | |||
87 | const videoKeysToKeep = [ | ||
88 | 'tags', | ||
89 | 'uuid', | ||
90 | 'id', | ||
91 | 'uuid', | ||
92 | 'createdAt', | ||
93 | 'updatedAt', | ||
94 | 'publishedAt', | ||
95 | 'category', | ||
96 | 'licence', | ||
97 | 'language', | ||
98 | 'privacy', | ||
99 | 'description', | ||
100 | 'duration', | ||
101 | 'isLocal', | ||
102 | 'name', | ||
103 | 'thumbnailPath', | ||
104 | 'previewPath', | ||
105 | 'nsfw', | ||
106 | 'waitTranscoding', | ||
107 | 'account-id', | ||
108 | 'account-uuid', | ||
109 | 'account-name', | ||
110 | 'channel-id', | ||
111 | 'channel-uuid', | ||
112 | 'channel-name', | ||
113 | 'support', | ||
114 | 'commentsEnabled' | ||
115 | ] | ||
116 | class VideoAuditView extends EntityAuditView { | ||
117 | constructor (private video: VideoDetails) { | ||
118 | super(videoKeysToKeep, 'video', video) | ||
119 | } | ||
120 | } | ||
121 | |||
122 | const videoImportKeysToKeep = [ | ||
123 | 'id', | ||
124 | 'targetUrl', | ||
125 | 'video-name' | ||
126 | ] | ||
127 | class VideoImportAuditView extends EntityAuditView { | ||
128 | constructor (private videoImport: VideoImport) { | ||
129 | super(videoImportKeysToKeep, 'video-import', videoImport) | ||
130 | } | ||
131 | } | ||
132 | |||
133 | const commentKeysToKeep = [ | ||
134 | 'id', | ||
135 | 'text', | ||
136 | 'threadId', | ||
137 | 'inReplyToCommentId', | ||
138 | 'videoId', | ||
139 | 'createdAt', | ||
140 | 'updatedAt', | ||
141 | 'totalReplies', | ||
142 | 'account-id', | ||
143 | 'account-uuid', | ||
144 | 'account-name' | ||
145 | ] | ||
146 | class CommentAuditView extends EntityAuditView { | ||
147 | constructor (private comment: VideoComment) { | ||
148 | super(commentKeysToKeep, 'comment', comment) | ||
149 | } | ||
150 | } | ||
151 | |||
152 | const userKeysToKeep = [ | ||
153 | 'id', | ||
154 | 'username', | ||
155 | 'email', | ||
156 | 'nsfwPolicy', | ||
157 | 'autoPlayVideo', | ||
158 | 'role', | ||
159 | 'videoQuota', | ||
160 | 'createdAt', | ||
161 | 'account-id', | ||
162 | 'account-uuid', | ||
163 | 'account-name', | ||
164 | 'account-followingCount', | ||
165 | 'account-followersCount', | ||
166 | 'account-createdAt', | ||
167 | 'account-updatedAt', | ||
168 | 'account-avatar-path', | ||
169 | 'account-avatar-createdAt', | ||
170 | 'account-avatar-updatedAt', | ||
171 | 'account-displayName', | ||
172 | 'account-description', | ||
173 | 'videoChannels' | ||
174 | ] | ||
175 | class UserAuditView extends EntityAuditView { | ||
176 | constructor (private user: User) { | ||
177 | super(userKeysToKeep, 'user', user) | ||
178 | } | ||
179 | } | ||
180 | |||
181 | const channelKeysToKeep = [ | ||
182 | 'id', | ||
183 | 'uuid', | ||
184 | 'name', | ||
185 | 'followingCount', | ||
186 | 'followersCount', | ||
187 | 'createdAt', | ||
188 | 'updatedAt', | ||
189 | 'avatar-path', | ||
190 | 'avatar-createdAt', | ||
191 | 'avatar-updatedAt', | ||
192 | 'displayName', | ||
193 | 'description', | ||
194 | 'support', | ||
195 | 'isLocal', | ||
196 | 'ownerAccount-id', | ||
197 | 'ownerAccount-uuid', | ||
198 | 'ownerAccount-name', | ||
199 | 'ownerAccount-displayedName' | ||
200 | ] | ||
201 | class VideoChannelAuditView extends EntityAuditView { | ||
202 | constructor (private channel: VideoChannel) { | ||
203 | super(channelKeysToKeep, 'channel', channel) | ||
204 | } | ||
205 | } | ||
206 | |||
207 | const videoAbuseKeysToKeep = [ | ||
208 | 'id', | ||
209 | 'reason', | ||
210 | 'reporterAccount', | ||
211 | 'video-id', | ||
212 | 'video-name', | ||
213 | 'video-uuid', | ||
214 | 'createdAt' | ||
215 | ] | ||
216 | class VideoAbuseAuditView extends EntityAuditView { | ||
217 | constructor (private videoAbuse: VideoAbuse) { | ||
218 | super(videoAbuseKeysToKeep, 'abuse', videoAbuse) | ||
219 | } | ||
220 | } | ||
221 | |||
222 | const customConfigKeysToKeep = [ | ||
223 | 'instance-name', | ||
224 | 'instance-shortDescription', | ||
225 | 'instance-description', | ||
226 | 'instance-terms', | ||
227 | 'instance-defaultClientRoute', | ||
228 | 'instance-defaultNSFWPolicy', | ||
229 | 'instance-customizations-javascript', | ||
230 | 'instance-customizations-css', | ||
231 | 'services-twitter-username', | ||
232 | 'services-twitter-whitelisted', | ||
233 | 'cache-previews-size', | ||
234 | 'cache-captions-size', | ||
235 | 'signup-enabled', | ||
236 | 'signup-limit', | ||
237 | 'admin-email', | ||
238 | 'user-videoQuota', | ||
239 | 'transcoding-enabled', | ||
240 | 'transcoding-threads', | ||
241 | 'transcoding-resolutions' | ||
242 | ] | ||
243 | class CustomConfigAuditView extends EntityAuditView { | ||
244 | constructor (customConfig: CustomConfig) { | ||
245 | const infos: any = customConfig | ||
246 | const resolutionsDict = infos.transcoding.resolutions | ||
247 | const resolutionsArray = [] | ||
248 | Object.entries(resolutionsDict).forEach(([resolution, isEnabled]) => { | ||
249 | if (isEnabled) resolutionsArray.push(resolution) | ||
250 | }) | ||
251 | Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } }) | ||
252 | super(customConfigKeysToKeep, 'config', infos) | ||
253 | } | ||
254 | } | ||
255 | |||
256 | export { | ||
257 | auditLoggerFactory, | ||
258 | VideoImportAuditView, | ||
259 | VideoChannelAuditView, | ||
260 | CommentAuditView, | ||
261 | UserAuditView, | ||
262 | VideoAuditView, | ||
263 | VideoAbuseAuditView, | ||
264 | CustomConfigAuditView | ||
265 | } | ||
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 2951aef1e..884206aad 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts | |||
@@ -58,7 +58,7 @@ function escapeHTML (stringParam) { | |||
58 | '<': '<', | 58 | '<': '<', |
59 | '>': '>', | 59 | '>': '>', |
60 | '"': '"', | 60 | '"': '"', |
61 | "'": ''', | 61 | '\'': ''', |
62 | '/': '/', | 62 | '/': '/', |
63 | '`': '`', | 63 | '`': '`', |
64 | '=': '=' | 64 | '=': '=' |
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index d97bbd2a9..b8075f3c7 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) { | |||
45 | } | 45 | } |
46 | 46 | ||
47 | function sanitizeAndCheckVideoTorrentObject (video: any) { | 47 | function sanitizeAndCheckVideoTorrentObject (video: any) { |
48 | if (video.type !== 'Video') return false | 48 | if (!video || video.type !== 'Video') return false |
49 | 49 | ||
50 | if (!setValidRemoteTags(video)) return false | 50 | if (!setValidRemoteTags(video)) return false |
51 | if (!setValidRemoteVideoUrls(video)) return false | 51 | if (!setValidRemoteVideoUrls(video)) return false |
@@ -153,7 +153,8 @@ function isRemoteVideoUrlValid (url: any) { | |||
153 | ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && | 153 | ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && |
154 | isActivityPubUrlValid(url.href) && | 154 | isActivityPubUrlValid(url.href) && |
155 | validator.isInt(url.width + '', { min: 0 }) && | 155 | validator.isInt(url.width + '', { min: 0 }) && |
156 | validator.isInt(url.size + '', { min: 0 }) | 156 | validator.isInt(url.size + '', { min: 0 }) && |
157 | (!url.fps || validator.isInt(url.fps + '', { min: 0 })) | ||
157 | ) || | 158 | ) || |
158 | ( | 159 | ( |
159 | ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && | 160 | ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && |
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts new file mode 100644 index 000000000..d8b9bfaff --- /dev/null +++ b/server/helpers/custom-validators/video-imports.ts | |||
@@ -0,0 +1,49 @@ | |||
1 | import 'express-validator' | ||
2 | import 'multer' | ||
3 | import * as validator from 'validator' | ||
4 | import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' | ||
5 | import { exists } from './misc' | ||
6 | import * as express from 'express' | ||
7 | import { VideoChannelModel } from '../../models/video/video-channel' | ||
8 | import { VideoImportModel } from '../../models/video/video-import' | ||
9 | |||
10 | function isVideoImportTargetUrlValid (url: string) { | ||
11 | const isURLOptions = { | ||
12 | require_host: true, | ||
13 | require_tld: true, | ||
14 | require_protocol: true, | ||
15 | require_valid_protocol: true, | ||
16 | protocols: [ 'http', 'https' ] | ||
17 | } | ||
18 | |||
19 | return exists(url) && | ||
20 | validator.isURL('' + url, isURLOptions) && | ||
21 | validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) | ||
22 | } | ||
23 | |||
24 | function isVideoImportStateValid (value: any) { | ||
25 | return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined | ||
26 | } | ||
27 | |||
28 | async function isVideoImportExist (id: number, res: express.Response) { | ||
29 | const videoImport = await VideoImportModel.loadAndPopulateVideo(id) | ||
30 | |||
31 | if (!videoImport) { | ||
32 | res.status(404) | ||
33 | .json({ error: 'Video import not found' }) | ||
34 | .end() | ||
35 | |||
36 | return false | ||
37 | } | ||
38 | |||
39 | res.locals.videoImport = videoImport | ||
40 | return true | ||
41 | } | ||
42 | |||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | export { | ||
46 | isVideoImportStateValid, | ||
47 | isVideoImportTargetUrlValid, | ||
48 | isVideoImportExist | ||
49 | } | ||
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index f0623c88b..ced56b82d 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts | |||
@@ -1,10 +1,11 @@ | |||
1 | import * as ffmpeg from 'fluent-ffmpeg' | 1 | import * as ffmpeg from 'fluent-ffmpeg' |
2 | import { join } from 'path' | 2 | import { join } from 'path' |
3 | import { VideoResolution } from '../../shared/models/videos' | 3 | import { VideoResolution } from '../../shared/models/videos' |
4 | import { CONFIG, VIDEO_TRANSCODING_FPS } from '../initializers' | 4 | import { CONFIG, VIDEO_TRANSCODING_FPS, FFMPEG_NICE } from '../initializers' |
5 | import { unlinkPromise } from './core-utils' | 5 | import { unlinkPromise } from './core-utils' |
6 | import { processImage } from './image-utils' | 6 | import { processImage } from './image-utils' |
7 | import { logger } from './logger' | 7 | import { logger } from './logger' |
8 | import { checkFFmpegEncoders } from '../initializers/checker' | ||
8 | 9 | ||
9 | async function getVideoFileResolution (path: string) { | 10 | async function getVideoFileResolution (path: string) { |
10 | const videoStream = await getVideoFileStream(path) | 11 | const videoStream = await getVideoFileStream(path) |
@@ -55,7 +56,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima | |||
55 | 56 | ||
56 | try { | 57 | try { |
57 | await new Promise<string>((res, rej) => { | 58 | await new Promise<string>((res, rej) => { |
58 | ffmpeg(fromPath) | 59 | ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL }) |
59 | .on('error', rej) | 60 | .on('error', rej) |
60 | .on('end', () => res(imageName)) | 61 | .on('end', () => res(imageName)) |
61 | .thumbnail(options) | 62 | .thumbnail(options) |
@@ -83,14 +84,14 @@ type TranscodeOptions = { | |||
83 | 84 | ||
84 | function transcode (options: TranscodeOptions) { | 85 | function transcode (options: TranscodeOptions) { |
85 | return new Promise<void>(async (res, rej) => { | 86 | return new Promise<void>(async (res, rej) => { |
86 | let command = ffmpeg(options.inputPath) | 87 | let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING }) |
87 | .output(options.outputPath) | 88 | .output(options.outputPath) |
88 | .videoCodec('libx264') | 89 | .preset(standard) |
89 | .outputOption('-threads ' + CONFIG.TRANSCODING.THREADS) | 90 | |
90 | .outputOption('-movflags faststart') | 91 | if (CONFIG.TRANSCODING.THREADS > 0) { |
91 | .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorythm, 16 is optimal B-frames for it | 92 | // if we don't set any threads ffmpeg will chose automatically |
92 | .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16 | 93 | command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS) |
93 | // .outputOption('-crf 18') | 94 | } |
94 | 95 | ||
95 | let fps = await getVideoFileFPS(options.inputPath) | 96 | let fps = await getVideoFileFPS(options.inputPath) |
96 | if (options.resolution !== undefined) { | 97 | if (options.resolution !== undefined) { |
@@ -132,7 +133,8 @@ export { | |||
132 | getDurationFromVideoFile, | 133 | getDurationFromVideoFile, |
133 | generateImageFromVideoFile, | 134 | generateImageFromVideoFile, |
134 | transcode, | 135 | transcode, |
135 | getVideoFileFPS | 136 | getVideoFileFPS, |
137 | audio | ||
136 | } | 138 | } |
137 | 139 | ||
138 | // --------------------------------------------------------------------------- | 140 | // --------------------------------------------------------------------------- |
@@ -149,3 +151,136 @@ function getVideoFileStream (path: string) { | |||
149 | }) | 151 | }) |
150 | }) | 152 | }) |
151 | } | 153 | } |
154 | |||
155 | /** | ||
156 | * A slightly customised version of the 'veryfast' x264 preset | ||
157 | * | ||
158 | * The veryfast preset is right in the sweet spot of performance | ||
159 | * and quality. Superfast and ultrafast will give you better | ||
160 | * performance, but then quality is noticeably worse. | ||
161 | */ | ||
162 | function veryfast (_ffmpeg) { | ||
163 | _ffmpeg | ||
164 | .preset(standard) | ||
165 | .outputOption('-preset:v veryfast') | ||
166 | .outputOption(['--aq-mode=2', '--aq-strength=1.3']) | ||
167 | /* | ||
168 | MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html | ||
169 | Our target situation is closer to a livestream than a stream, | ||
170 | since we want to reduce as much a possible the encoding burden, | ||
171 | altough not to the point of a livestream where there is a hard | ||
172 | constraint on the frames per second to be encoded. | ||
173 | |||
174 | why '--aq-mode=2 --aq-strength=1.3' instead of '-profile:v main'? | ||
175 | Make up for most of the loss of grain and macroblocking | ||
176 | with less computing power. | ||
177 | */ | ||
178 | } | ||
179 | |||
180 | /** | ||
181 | * A preset optimised for a stillimage audio video | ||
182 | */ | ||
183 | function audio (_ffmpeg) { | ||
184 | _ffmpeg | ||
185 | .preset(veryfast) | ||
186 | .outputOption('-tune stillimage') | ||
187 | } | ||
188 | |||
189 | /** | ||
190 | * A toolbox to play with audio | ||
191 | */ | ||
192 | namespace audio { | ||
193 | export const get = (_ffmpeg, pos: number | string = 0) => { | ||
194 | // without position, ffprobe considers the last input only | ||
195 | // we make it consider the first input only | ||
196 | // if you pass a file path to pos, then ffprobe acts on that file directly | ||
197 | return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => { | ||
198 | _ffmpeg.ffprobe(pos, (err,data) => { | ||
199 | if (err) return rej(err) | ||
200 | |||
201 | if ('streams' in data) { | ||
202 | const audioStream = data['streams'].find(stream => stream['codec_type'] === 'audio') | ||
203 | if (audioStream) { | ||
204 | return res({ | ||
205 | absolutePath: data.format.filename, | ||
206 | audioStream | ||
207 | }) | ||
208 | } | ||
209 | } | ||
210 | return res({ absolutePath: data.format.filename }) | ||
211 | }) | ||
212 | }) | ||
213 | } | ||
214 | |||
215 | export namespace bitrate { | ||
216 | export const baseKbitrate = 384 | ||
217 | |||
218 | const toBits = (kbits: number): number => { return kbits * 8000 } | ||
219 | |||
220 | export const aac = (bitrate: number): number => { | ||
221 | switch (true) { | ||
222 | case bitrate > toBits(baseKbitrate): | ||
223 | return baseKbitrate | ||
224 | default: | ||
225 | return -1 // we interpret it as a signal to copy the audio stream as is | ||
226 | } | ||
227 | } | ||
228 | |||
229 | export const mp3 = (bitrate: number): number => { | ||
230 | /* | ||
231 | a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. | ||
232 | That's why, when using aac, we can go to lower kbit/sec. The equivalences | ||
233 | made here are not made to be accurate, especially with good mp3 encoders. | ||
234 | */ | ||
235 | switch (true) { | ||
236 | case bitrate <= toBits(192): | ||
237 | return 128 | ||
238 | case bitrate <= toBits(384): | ||
239 | return 256 | ||
240 | default: | ||
241 | return baseKbitrate | ||
242 | } | ||
243 | } | ||
244 | } | ||
245 | } | ||
246 | |||
247 | /** | ||
248 | * Standard profile, with variable bitrate audio and faststart. | ||
249 | * | ||
250 | * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel | ||
251 | * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr | ||
252 | */ | ||
253 | async function standard (_ffmpeg) { | ||
254 | let _bitrate = audio.bitrate.baseKbitrate | ||
255 | let localFfmpeg = _ffmpeg | ||
256 | .format('mp4') | ||
257 | .videoCodec('libx264') | ||
258 | .outputOption('-level 3.1') // 3.1 is the minimal ressource allocation for our highest supported resolution | ||
259 | .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorythm, 16 is optimal B-frames for it | ||
260 | .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16 | ||
261 | .outputOption('-map_metadata -1') // strip all metadata | ||
262 | .outputOption('-movflags faststart') | ||
263 | const _audio = await audio.get(localFfmpeg) | ||
264 | |||
265 | if (!_audio.audioStream) { | ||
266 | return localFfmpeg.noAudio() | ||
267 | } | ||
268 | |||
269 | // we try to reduce the ceiling bitrate by making rough correspondances of bitrates | ||
270 | // of course this is far from perfect, but it might save some space in the end | ||
271 | if (audio.bitrate[_audio.audioStream['codec_name']]) { | ||
272 | _bitrate = audio.bitrate[_audio.audioStream['codec_name']](_audio.audioStream['bit_rate']) | ||
273 | if (_bitrate === -1) { | ||
274 | return localFfmpeg.audioCodec('copy') | ||
275 | } | ||
276 | } | ||
277 | |||
278 | // we favor VBR, if a good AAC encoder is available | ||
279 | if ((await checkFFmpegEncoders()).get('libfdk_aac')) { | ||
280 | return localFfmpeg | ||
281 | .audioCodec('libfdk_aac') | ||
282 | .audioQuality(5) | ||
283 | } | ||
284 | |||
285 | return localFfmpeg.audioBitrate(_bitrate) | ||
286 | } | ||
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 6d369a8fb..480c5b49e 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts | |||
@@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) { | |||
22 | } | 22 | } |
23 | 23 | ||
24 | const consoleLoggerFormat = winston.format.printf(info => { | 24 | const consoleLoggerFormat = winston.format.printf(info => { |
25 | let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) | 25 | let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2) |
26 | if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' | 26 | if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' |
27 | else additionalInfos = ' ' + additionalInfos | 27 | else additionalInfos = ' ' + additionalInfos |
28 | 28 | ||
@@ -96,13 +96,13 @@ const bunyanLogger = { | |||
96 | error: bunyanLogFactory('error'), | 96 | error: bunyanLogFactory('error'), |
97 | fatal: bunyanLogFactory('error') | 97 | fatal: bunyanLogFactory('error') |
98 | } | 98 | } |
99 | |||
100 | // --------------------------------------------------------------------------- | 99 | // --------------------------------------------------------------------------- |
101 | 100 | ||
102 | export { | 101 | export { |
103 | timestampFormatter, | 102 | timestampFormatter, |
104 | labelFormatter, | 103 | labelFormatter, |
105 | consoleLoggerFormat, | 104 | consoleLoggerFormat, |
105 | jsonLoggerFormat, | ||
106 | logger, | 106 | logger, |
107 | bunyanLogger | 107 | bunyanLogger |
108 | } | 108 | } |
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index cfb427570..7abcec5d7 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts | |||
@@ -6,11 +6,35 @@ import { CONFIG } from '../initializers' | |||
6 | import { UserModel } from '../models/account/user' | 6 | import { UserModel } from '../models/account/user' |
7 | import { ActorModel } from '../models/activitypub/actor' | 7 | import { ActorModel } from '../models/activitypub/actor' |
8 | import { ApplicationModel } from '../models/application/application' | 8 | import { ApplicationModel } from '../models/application/application' |
9 | import { pseudoRandomBytesPromise } from './core-utils' | 9 | import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils' |
10 | import { logger } from './logger' | 10 | import { logger } from './logger' |
11 | import { isArray } from './custom-validators/misc' | ||
11 | 12 | ||
12 | const isCidr = require('is-cidr') | 13 | const isCidr = require('is-cidr') |
13 | 14 | ||
15 | function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[] }) { | ||
16 | const files = req.files | ||
17 | |||
18 | if (!files) return | ||
19 | |||
20 | if (isArray(files)) { | ||
21 | (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path)) | ||
22 | return | ||
23 | } | ||
24 | |||
25 | for (const key of Object.keys(files)) { | ||
26 | const file = files[key] | ||
27 | |||
28 | if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) | ||
29 | else deleteFileAsync(file.path) | ||
30 | } | ||
31 | } | ||
32 | |||
33 | function deleteFileAsync (path: string) { | ||
34 | unlinkPromise(path) | ||
35 | .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) | ||
36 | } | ||
37 | |||
14 | async function generateRandomString (size: number) { | 38 | async function generateRandomString (size: number) { |
15 | const raw = await pseudoRandomBytesPromise(size) | 39 | const raw = await pseudoRandomBytesPromise(size) |
16 | 40 | ||
@@ -162,6 +186,8 @@ type SortType = { sortModel: any, sortValue: string } | |||
162 | // --------------------------------------------------------------------------- | 186 | // --------------------------------------------------------------------------- |
163 | 187 | ||
164 | export { | 188 | export { |
189 | cleanUpReqFiles, | ||
190 | deleteFileAsync, | ||
165 | generateRandomString, | 191 | generateRandomString, |
166 | getFormattedObjects, | 192 | getFormattedObjects, |
167 | isSignupAllowed, | 193 | isSignupAllowed, |
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts new file mode 100644 index 000000000..c59ab9de0 --- /dev/null +++ b/server/helpers/youtube-dl.ts | |||
@@ -0,0 +1,142 @@ | |||
1 | import * as youtubeDL from 'youtube-dl' | ||
2 | import { truncate } from 'lodash' | ||
3 | import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers' | ||
4 | import { join } from 'path' | ||
5 | import * as crypto from 'crypto' | ||
6 | import { logger } from './logger' | ||
7 | |||
8 | export type YoutubeDLInfo = { | ||
9 | name: string | ||
10 | description: string | ||
11 | category: number | ||
12 | licence: number | ||
13 | nsfw: boolean | ||
14 | tags: string[] | ||
15 | thumbnailUrl: string | ||
16 | } | ||
17 | |||
18 | function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> { | ||
19 | return new Promise<YoutubeDLInfo>((res, rej) => { | ||
20 | const options = [ '-j', '--flat-playlist' ] | ||
21 | |||
22 | youtubeDL.getInfo(url, options, (err, info) => { | ||
23 | if (err) return rej(err) | ||
24 | |||
25 | const obj = normalizeObject(info) | ||
26 | |||
27 | return res(buildVideoInfo(obj)) | ||
28 | }) | ||
29 | }) | ||
30 | } | ||
31 | |||
32 | function downloadYoutubeDLVideo (url: string) { | ||
33 | const hash = crypto.createHash('sha256').update(url).digest('hex') | ||
34 | const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4') | ||
35 | |||
36 | logger.info('Importing video %s', url) | ||
37 | |||
38 | const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] | ||
39 | |||
40 | return new Promise<string>((res, rej) => { | ||
41 | youtubeDL.exec(url, options, async (err, output) => { | ||
42 | if (err) return rej(err) | ||
43 | |||
44 | return res(path) | ||
45 | }) | ||
46 | }) | ||
47 | } | ||
48 | |||
49 | // --------------------------------------------------------------------------- | ||
50 | |||
51 | export { | ||
52 | downloadYoutubeDLVideo, | ||
53 | getYoutubeDLInfo | ||
54 | } | ||
55 | |||
56 | // --------------------------------------------------------------------------- | ||
57 | |||
58 | function normalizeObject (obj: any) { | ||
59 | const newObj: any = {} | ||
60 | |||
61 | for (const key of Object.keys(obj)) { | ||
62 | // Deprecated key | ||
63 | if (key === 'resolution') continue | ||
64 | |||
65 | const value = obj[key] | ||
66 | |||
67 | if (typeof value === 'string') { | ||
68 | newObj[key] = value.normalize() | ||
69 | } else { | ||
70 | newObj[key] = value | ||
71 | } | ||
72 | } | ||
73 | |||
74 | return newObj | ||
75 | } | ||
76 | |||
77 | function buildVideoInfo (obj: any) { | ||
78 | return { | ||
79 | name: titleTruncation(obj.title), | ||
80 | description: descriptionTruncation(obj.description), | ||
81 | category: getCategory(obj.categories), | ||
82 | licence: getLicence(obj.license), | ||
83 | nsfw: isNSFW(obj), | ||
84 | tags: getTags(obj.tags), | ||
85 | thumbnailUrl: obj.thumbnail || undefined | ||
86 | } | ||
87 | } | ||
88 | |||
89 | function titleTruncation (title: string) { | ||
90 | return truncate(title, { | ||
91 | 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max, | ||
92 | 'separator': /,? +/, | ||
93 | 'omission': ' […]' | ||
94 | }) | ||
95 | } | ||
96 | |||
97 | function descriptionTruncation (description: string) { | ||
98 | if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined | ||
99 | |||
100 | return truncate(description, { | ||
101 | 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, | ||
102 | 'separator': /,? +/, | ||
103 | 'omission': ' […]' | ||
104 | }) | ||
105 | } | ||
106 | |||
107 | function isNSFW (info: any) { | ||
108 | return info.age_limit && info.age_limit >= 16 | ||
109 | } | ||
110 | |||
111 | function getTags (tags: any) { | ||
112 | if (Array.isArray(tags) === false) return [] | ||
113 | |||
114 | return tags | ||
115 | .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) | ||
116 | .map(t => t.normalize()) | ||
117 | .slice(0, 5) | ||
118 | } | ||
119 | |||
120 | function getLicence (licence: string) { | ||
121 | if (!licence) return undefined | ||
122 | |||
123 | if (licence.indexOf('Creative Commons Attribution') !== -1) return 1 | ||
124 | |||
125 | return undefined | ||
126 | } | ||
127 | |||
128 | function getCategory (categories: string[]) { | ||
129 | if (!categories) return undefined | ||
130 | |||
131 | const categoryString = categories[0] | ||
132 | if (!categoryString || typeof categoryString !== 'string') return undefined | ||
133 | |||
134 | if (categoryString === 'News & Politics') return 11 | ||
135 | |||
136 | for (const key of Object.keys(VIDEO_CATEGORIES)) { | ||
137 | const category = VIDEO_CATEGORIES[key] | ||
138 | if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) | ||
139 | } | ||
140 | |||
141 | return undefined | ||
142 | } | ||
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 52a1aeb50..608123607 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts | |||
@@ -43,7 +43,7 @@ function checkMissedConfig () { | |||
43 | const required = [ 'listen.port', 'listen.hostname', | 43 | const required = [ 'listen.port', 'listen.hostname', |
44 | 'webserver.https', 'webserver.hostname', 'webserver.port', | 44 | 'webserver.https', 'webserver.hostname', 'webserver.port', |
45 | 'trust_proxy', | 45 | 'trust_proxy', |
46 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', | 46 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', |
47 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', | 47 | 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', |
48 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', | 48 | 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', |
49 | 'log.level', | 49 | 'log.level', |
@@ -51,6 +51,7 @@ function checkMissedConfig () { | |||
51 | 'cache.previews.size', 'admin.email', | 51 | 'cache.previews.size', 'admin.email', |
52 | 'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', | 52 | 'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', |
53 | 'transcoding.enabled', 'transcoding.threads', | 53 | 'transcoding.enabled', 'transcoding.threads', |
54 | 'import.videos.http.enabled', | ||
54 | 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', | 55 | 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', |
55 | 'instance.default_nsfw_policy', 'instance.robots', | 56 | 'instance.default_nsfw_policy', 'instance.robots', |
56 | 'services.twitter.username', 'services.twitter.whitelisted' | 57 | 'services.twitter.username', 'services.twitter.whitelisted' |
@@ -84,11 +85,11 @@ function checkMissedConfig () { | |||
84 | async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { | 85 | async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { |
85 | const Ffmpeg = require('fluent-ffmpeg') | 86 | const Ffmpeg = require('fluent-ffmpeg') |
86 | const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) | 87 | const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) |
87 | |||
88 | const codecs = await getAvailableCodecsPromise() | 88 | const codecs = await getAvailableCodecsPromise() |
89 | const canEncode = [ 'libx264' ] | ||
90 | |||
89 | if (CONFIG.TRANSCODING.ENABLED === false) return undefined | 91 | if (CONFIG.TRANSCODING.ENABLED === false) return undefined |
90 | 92 | ||
91 | const canEncode = [ 'libx264' ] | ||
92 | for (const codec of canEncode) { | 93 | for (const codec of canEncode) { |
93 | if (codecs[codec] === undefined) { | 94 | if (codecs[codec] === undefined) { |
94 | throw new Error('Unknown codec ' + codec + ' in FFmpeg.') | 95 | throw new Error('Unknown codec ' + codec + ' in FFmpeg.') |
@@ -98,6 +99,29 @@ async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { | |||
98 | throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg') | 99 | throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg') |
99 | } | 100 | } |
100 | } | 101 | } |
102 | |||
103 | checkFFmpegEncoders() | ||
104 | } | ||
105 | |||
106 | // Optional encoders, if present, can be used to improve transcoding | ||
107 | // Here we ask ffmpeg if it detects their presence on the system, so that we can later use them | ||
108 | let supportedOptionalEncoders: Map<string, boolean> | ||
109 | async function checkFFmpegEncoders (): Promise<Map<string, boolean>> { | ||
110 | if (supportedOptionalEncoders !== undefined) { | ||
111 | return supportedOptionalEncoders | ||
112 | } | ||
113 | |||
114 | const Ffmpeg = require('fluent-ffmpeg') | ||
115 | const getAvailableEncodersPromise = promisify0(Ffmpeg.getAvailableEncoders) | ||
116 | const encoders = await getAvailableEncodersPromise() | ||
117 | const optionalEncoders = [ 'libfdk_aac' ] | ||
118 | supportedOptionalEncoders = new Map<string, boolean>() | ||
119 | |||
120 | for (const encoder of optionalEncoders) { | ||
121 | supportedOptionalEncoders.set(encoder, | ||
122 | encoders[encoder] !== undefined | ||
123 | ) | ||
124 | } | ||
101 | } | 125 | } |
102 | 126 | ||
103 | // We get db by param to not import it in this file (import orders) | 127 | // We get db by param to not import it in this file (import orders) |
@@ -126,6 +150,7 @@ async function applicationExist () { | |||
126 | export { | 150 | export { |
127 | checkConfig, | 151 | checkConfig, |
128 | checkFFmpeg, | 152 | checkFFmpeg, |
153 | checkFFmpegEncoders, | ||
129 | checkMissedConfig, | 154 | checkMissedConfig, |
130 | clientsExist, | 155 | clientsExist, |
131 | usersExist, | 156 | usersExist, |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 3aa979668..069d9b2e8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos' | |||
8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' | 8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' |
9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' | 9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' |
10 | import { invert } from 'lodash' | 10 | import { invert } from 'lodash' |
11 | import { VideoImportState } from '../../shared/models/videos/video-import-state.enum' | ||
11 | 12 | ||
12 | // Use a variable to reload the configuration if we need | 13 | // Use a variable to reload the configuration if we need |
13 | let config: IConfig = require('config') | 14 | let config: IConfig = require('config') |
@@ -36,6 +37,7 @@ const SORTABLE_COLUMNS = { | |||
36 | VIDEO_ABUSES: [ 'id', 'createdAt' ], | 37 | VIDEO_ABUSES: [ 'id', 'createdAt' ], |
37 | VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], | 38 | VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], |
38 | VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ], | 39 | VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ], |
40 | VIDEO_IMPORTS: [ 'createdAt' ], | ||
39 | VIDEO_COMMENT_THREADS: [ 'createdAt' ], | 41 | VIDEO_COMMENT_THREADS: [ 'createdAt' ], |
40 | BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ], | 42 | BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ], |
41 | FOLLOWERS: [ 'createdAt' ], | 43 | FOLLOWERS: [ 'createdAt' ], |
@@ -85,6 +87,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = { | |||
85 | 'activitypub-follow': 5, | 87 | 'activitypub-follow': 5, |
86 | 'video-file-import': 1, | 88 | 'video-file-import': 1, |
87 | 'video-file': 1, | 89 | 'video-file': 1, |
90 | 'video-import': 1, | ||
88 | 'email': 5 | 91 | 'email': 5 |
89 | } | 92 | } |
90 | const JOB_CONCURRENCY: { [ id in JobType ]: number } = { | 93 | const JOB_CONCURRENCY: { [ id in JobType ]: number } = { |
@@ -94,18 +97,29 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = { | |||
94 | 'activitypub-follow': 3, | 97 | 'activitypub-follow': 3, |
95 | 'video-file-import': 1, | 98 | 'video-file-import': 1, |
96 | 'video-file': 1, | 99 | 'video-file': 1, |
100 | 'video-import': 1, | ||
97 | 'email': 5 | 101 | 'email': 5 |
98 | } | 102 | } |
103 | const JOB_TTL: { [ id in JobType ]: number } = { | ||
104 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | ||
105 | 'activitypub-http-unicast': 60000 * 10, // 10 minutes | ||
106 | 'activitypub-http-fetcher': 60000 * 10, // 10 minutes | ||
107 | 'activitypub-follow': 60000 * 10, // 10 minutes | ||
108 | 'video-file-import': 1000 * 3600, // 1 hour | ||
109 | 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long | ||
110 | 'video-import': 1000 * 3600 * 5, // 5 hours | ||
111 | 'email': 60000 * 10 // 10 minutes | ||
112 | } | ||
99 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job | 113 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job |
100 | const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds | 114 | const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds |
101 | const JOB_REQUEST_TTL = 60000 * 10 // 10 minutes | ||
102 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days | 115 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days |
103 | 116 | ||
104 | // 1 hour | 117 | // 1 hour |
105 | let SCHEDULER_INTERVALS_MS = { | 118 | let SCHEDULER_INTERVALS_MS = { |
106 | badActorFollow: 60000 * 60, // 1 hour | 119 | badActorFollow: 60000 * 60, // 1 hour |
107 | removeOldJobs: 60000 * 60, // 1 hour | 120 | removeOldJobs: 60000 * 60, // 1 hour |
108 | updateVideos: 60000 // 1 minute | 121 | updateVideos: 60000, // 1 minute |
122 | youtubeDLUpdate: 60000 * 60 * 24 // 1 day | ||
109 | } | 123 | } |
110 | 124 | ||
111 | // --------------------------------------------------------------------------- | 125 | // --------------------------------------------------------------------------- |
@@ -121,7 +135,10 @@ const CONFIG = { | |||
121 | HOSTNAME: config.get<string>('database.hostname'), | 135 | HOSTNAME: config.get<string>('database.hostname'), |
122 | PORT: config.get<number>('database.port'), | 136 | PORT: config.get<number>('database.port'), |
123 | USERNAME: config.get<string>('database.username'), | 137 | USERNAME: config.get<string>('database.username'), |
124 | PASSWORD: config.get<string>('database.password') | 138 | PASSWORD: config.get<string>('database.password'), |
139 | POOL: { | ||
140 | MAX: config.get<number>('database.pool.max') | ||
141 | } | ||
125 | }, | 142 | }, |
126 | REDIS: { | 143 | REDIS: { |
127 | HOSTNAME: config.has('redis.hostname') ? config.get<string>('redis.hostname') : null, | 144 | HOSTNAME: config.has('redis.hostname') ? config.get<string>('redis.hostname') : null, |
@@ -189,6 +206,13 @@ const CONFIG = { | |||
189 | get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } | 206 | get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } |
190 | } | 207 | } |
191 | }, | 208 | }, |
209 | IMPORT: { | ||
210 | VIDEOS: { | ||
211 | HTTP: { | ||
212 | get ENABLED () { return config.get<boolean>('import.videos.http.enabled') } | ||
213 | } | ||
214 | } | ||
215 | }, | ||
192 | CACHE: { | 216 | CACHE: { |
193 | PREVIEWS: { | 217 | PREVIEWS: { |
194 | get SIZE () { return config.get<number>('cache.previews.size') } | 218 | get SIZE () { return config.get<number>('cache.previews.size') } |
@@ -245,6 +269,9 @@ const CONSTRAINTS_FIELDS = { | |||
245 | } | 269 | } |
246 | } | 270 | } |
247 | }, | 271 | }, |
272 | VIDEO_IMPORTS: { | ||
273 | URL: { min: 3, max: 2000 } // Length | ||
274 | }, | ||
248 | VIDEOS: { | 275 | VIDEOS: { |
249 | NAME: { min: 3, max: 120 }, // Length | 276 | NAME: { min: 3, max: 120 }, // Length |
250 | LANGUAGE: { min: 1, max: 10 }, // Length | 277 | LANGUAGE: { min: 1, max: 10 }, // Length |
@@ -259,7 +286,7 @@ const CONSTRAINTS_FIELDS = { | |||
259 | }, | 286 | }, |
260 | EXTNAME: [ '.mp4', '.ogv', '.webm' ], | 287 | EXTNAME: [ '.mp4', '.ogv', '.webm' ], |
261 | INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 | 288 | INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 |
262 | DURATION: { min: 1 }, // Number | 289 | DURATION: { min: 0 }, // Number |
263 | TAGS: { min: 0, max: 5 }, // Number of total tags | 290 | TAGS: { min: 0, max: 5 }, // Number of total tags |
264 | TAG: { min: 2, max: 30 }, // Length | 291 | TAG: { min: 2, max: 30 }, // Length |
265 | THUMBNAIL: { min: 2, max: 30 }, | 292 | THUMBNAIL: { min: 2, max: 30 }, |
@@ -313,6 +340,11 @@ const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = { | |||
313 | DISLIKE: 'dislike' | 340 | DISLIKE: 'dislike' |
314 | } | 341 | } |
315 | 342 | ||
343 | const FFMPEG_NICE: { [ id: string ]: number } = { | ||
344 | THUMBNAIL: 2, // 2 just for don't blocking servers | ||
345 | TRANSCODING: 15 | ||
346 | } | ||
347 | |||
316 | const VIDEO_CATEGORIES = { | 348 | const VIDEO_CATEGORIES = { |
317 | 1: 'Music', | 349 | 1: 'Music', |
318 | 2: 'Films', | 350 | 2: 'Films', |
@@ -355,7 +387,14 @@ const VIDEO_PRIVACIES = { | |||
355 | 387 | ||
356 | const VIDEO_STATES = { | 388 | const VIDEO_STATES = { |
357 | [VideoState.PUBLISHED]: 'Published', | 389 | [VideoState.PUBLISHED]: 'Published', |
358 | [VideoState.TO_TRANSCODE]: 'To transcode' | 390 | [VideoState.TO_TRANSCODE]: 'To transcode', |
391 | [VideoState.TO_IMPORT]: 'To import' | ||
392 | } | ||
393 | |||
394 | const VIDEO_IMPORT_STATES = { | ||
395 | [VideoImportState.FAILED]: 'Failed', | ||
396 | [VideoImportState.PENDING]: 'Pending', | ||
397 | [VideoImportState.SUCCESS]: 'Success' | ||
359 | } | 398 | } |
360 | 399 | ||
361 | const VIDEO_MIMETYPE_EXT = { | 400 | const VIDEO_MIMETYPE_EXT = { |
@@ -553,6 +592,7 @@ export { | |||
553 | ROUTE_CACHE_LIFETIME, | 592 | ROUTE_CACHE_LIFETIME, |
554 | SORTABLE_COLUMNS, | 593 | SORTABLE_COLUMNS, |
555 | FEEDS, | 594 | FEEDS, |
595 | JOB_TTL, | ||
556 | NSFW_POLICY_TYPES, | 596 | NSFW_POLICY_TYPES, |
557 | STATIC_MAX_AGE, | 597 | STATIC_MAX_AGE, |
558 | STATIC_PATHS, | 598 | STATIC_PATHS, |
@@ -567,8 +607,8 @@ export { | |||
567 | VIDEO_RATE_TYPES, | 607 | VIDEO_RATE_TYPES, |
568 | VIDEO_MIMETYPE_EXT, | 608 | VIDEO_MIMETYPE_EXT, |
569 | VIDEO_TRANSCODING_FPS, | 609 | VIDEO_TRANSCODING_FPS, |
610 | FFMPEG_NICE, | ||
570 | JOB_REQUEST_TIMEOUT, | 611 | JOB_REQUEST_TIMEOUT, |
571 | JOB_REQUEST_TTL, | ||
572 | USER_PASSWORD_RESET_LIFETIME, | 612 | USER_PASSWORD_RESET_LIFETIME, |
573 | IMAGE_MIMETYPE_EXT, | 613 | IMAGE_MIMETYPE_EXT, |
574 | SCHEDULER_INTERVALS_MS, | 614 | SCHEDULER_INTERVALS_MS, |
@@ -576,6 +616,7 @@ export { | |||
576 | RATES_LIMIT, | 616 | RATES_LIMIT, |
577 | VIDEO_EXT_MIMETYPE, | 617 | VIDEO_EXT_MIMETYPE, |
578 | JOB_COMPLETED_LIFETIME, | 618 | JOB_COMPLETED_LIFETIME, |
619 | VIDEO_IMPORT_STATES, | ||
579 | VIDEO_VIEW_LIFETIME, | 620 | VIDEO_VIEW_LIFETIME, |
580 | buildLanguages | 621 | buildLanguages |
581 | } | 622 | } |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 1a9ce5a61..0be752363 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag' | |||
24 | import { CONFIG } from './constants' | 24 | import { CONFIG } from './constants' |
25 | import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' | 25 | import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' |
26 | import { VideoCaptionModel } from '../models/video/video-caption' | 26 | import { VideoCaptionModel } from '../models/video/video-caption' |
27 | import { VideoImportModel } from '../models/video/video-import' | ||
27 | 28 | ||
28 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 29 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
29 | 30 | ||
@@ -32,6 +33,7 @@ const username = CONFIG.DATABASE.USERNAME | |||
32 | const password = CONFIG.DATABASE.PASSWORD | 33 | const password = CONFIG.DATABASE.PASSWORD |
33 | const host = CONFIG.DATABASE.HOSTNAME | 34 | const host = CONFIG.DATABASE.HOSTNAME |
34 | const port = CONFIG.DATABASE.PORT | 35 | const port = CONFIG.DATABASE.PORT |
36 | const poolMax = CONFIG.DATABASE.POOL.MAX | ||
35 | 37 | ||
36 | const sequelizeTypescript = new SequelizeTypescript({ | 38 | const sequelizeTypescript = new SequelizeTypescript({ |
37 | database: dbname, | 39 | database: dbname, |
@@ -40,6 +42,9 @@ const sequelizeTypescript = new SequelizeTypescript({ | |||
40 | port, | 42 | port, |
41 | username, | 43 | username, |
42 | password, | 44 | password, |
45 | pool: { | ||
46 | max: poolMax | ||
47 | }, | ||
43 | benchmark: isTestInstance(), | 48 | benchmark: isTestInstance(), |
44 | isolationLevel: SequelizeTypescript.Transaction.ISOLATION_LEVELS.SERIALIZABLE, | 49 | isolationLevel: SequelizeTypescript.Transaction.ISOLATION_LEVELS.SERIALIZABLE, |
45 | operatorsAliases: false, | 50 | operatorsAliases: false, |
@@ -77,7 +82,8 @@ async function initDatabaseModels (silent: boolean) { | |||
77 | VideoTagModel, | 82 | VideoTagModel, |
78 | VideoModel, | 83 | VideoModel, |
79 | VideoCommentModel, | 84 | VideoCommentModel, |
80 | ScheduleVideoUpdateModel | 85 | ScheduleVideoUpdateModel, |
86 | VideoImportModel | ||
81 | ]) | 87 | ]) |
82 | 88 | ||
83 | // Check extensions exist in the database | 89 | // Check extensions exist in the database |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 62791ff1b..82b661a03 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -108,7 +108,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
108 | await Promise.all(videoFileDestroyTasks) | 108 | await Promise.all(videoFileDestroyTasks) |
109 | 109 | ||
110 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject) | 110 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject) |
111 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f)) | 111 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) |
112 | await Promise.all(tasks) | 112 | await Promise.all(tasks) |
113 | 113 | ||
114 | // Update Tags | 114 | // Update Tags |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index b3fbf88d0..e2f46bd02 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -162,7 +162,8 @@ function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObje | |||
162 | infoHash: parsed.infoHash, | 162 | infoHash: parsed.infoHash, |
163 | resolution: fileUrl.width, | 163 | resolution: fileUrl.width, |
164 | size: fileUrl.size, | 164 | size: fileUrl.size, |
165 | videoId: videoCreated.id | 165 | videoId: videoCreated.id, |
166 | fps: fileUrl.fps | ||
166 | } as VideoFileModel | 167 | } as VideoFileModel |
167 | attributes.push(attribute) | 168 | attributes.push(attribute) |
168 | } | 169 | } |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts new file mode 100644 index 000000000..cdfe412cc --- /dev/null +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -0,0 +1,145 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { logger } from '../../../helpers/logger' | ||
3 | import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' | ||
4 | import { VideoImportModel } from '../../../models/video/video-import' | ||
5 | import { VideoImportState } from '../../../../shared/models/videos' | ||
6 | import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' | ||
7 | import { extname, join } from 'path' | ||
8 | import { VideoFileModel } from '../../../models/video/video-file' | ||
9 | import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils' | ||
10 | import { CONFIG, sequelizeTypescript } from '../../../initializers' | ||
11 | import { doRequestAndSaveToFile } from '../../../helpers/requests' | ||
12 | import { VideoState } from '../../../../shared' | ||
13 | import { JobQueue } from '../index' | ||
14 | import { federateVideoIfNeeded } from '../../activitypub' | ||
15 | import { VideoModel } from '../../../models/video/video' | ||
16 | |||
17 | export type VideoImportPayload = { | ||
18 | type: 'youtube-dl' | ||
19 | videoImportId: number | ||
20 | thumbnailUrl: string | ||
21 | downloadThumbnail: boolean | ||
22 | downloadPreview: boolean | ||
23 | } | ||
24 | |||
25 | async function processVideoImport (job: Bull.Job) { | ||
26 | const payload = job.data as VideoImportPayload | ||
27 | logger.info('Processing video import in job %d.', job.id) | ||
28 | |||
29 | const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) | ||
30 | if (!videoImport || !videoImport.Video) { | ||
31 | throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.') | ||
32 | } | ||
33 | |||
34 | let tempVideoPath: string | ||
35 | let videoDestFile: string | ||
36 | let videoFile: VideoFileModel | ||
37 | try { | ||
38 | // Download video from youtubeDL | ||
39 | tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl) | ||
40 | |||
41 | // Get information about this video | ||
42 | const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) | ||
43 | const fps = await getVideoFileFPS(tempVideoPath) | ||
44 | const stats = await statPromise(tempVideoPath) | ||
45 | const duration = await getDurationFromVideoFile(tempVideoPath) | ||
46 | |||
47 | // Create video file object in database | ||
48 | const videoFileData = { | ||
49 | extname: extname(tempVideoPath), | ||
50 | resolution: videoFileResolution, | ||
51 | size: stats.size, | ||
52 | fps, | ||
53 | videoId: videoImport.videoId | ||
54 | } | ||
55 | videoFile = new VideoFileModel(videoFileData) | ||
56 | // Import if the import fails, to clean files | ||
57 | videoImport.Video.VideoFiles = [ videoFile ] | ||
58 | |||
59 | // Move file | ||
60 | videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile)) | ||
61 | await renamePromise(tempVideoPath, videoDestFile) | ||
62 | tempVideoPath = null // This path is not used anymore | ||
63 | |||
64 | // Process thumbnail | ||
65 | if (payload.downloadThumbnail) { | ||
66 | if (payload.thumbnailUrl) { | ||
67 | const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName()) | ||
68 | await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath) | ||
69 | } else { | ||
70 | await videoImport.Video.createThumbnail(videoFile) | ||
71 | } | ||
72 | } | ||
73 | |||
74 | // Process preview | ||
75 | if (payload.downloadPreview) { | ||
76 | if (payload.thumbnailUrl) { | ||
77 | const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName()) | ||
78 | await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath) | ||
79 | } else { | ||
80 | await videoImport.Video.createPreview(videoFile) | ||
81 | } | ||
82 | } | ||
83 | |||
84 | // Create torrent | ||
85 | await videoImport.Video.createTorrentAndSetInfoHash(videoFile) | ||
86 | |||
87 | const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => { | ||
88 | // Refresh video | ||
89 | const video = await VideoModel.load(videoImport.videoId, t) | ||
90 | if (!video) throw new Error('Video linked to import ' + videoImport.videoId + ' does not exist anymore.') | ||
91 | videoImport.Video = video | ||
92 | |||
93 | const videoFileCreated = await videoFile.save({ transaction: t }) | ||
94 | video.VideoFiles = [ videoFileCreated ] | ||
95 | |||
96 | // Update video DB object | ||
97 | video.duration = duration | ||
98 | video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED | ||
99 | const videoUpdated = await video.save({ transaction: t }) | ||
100 | |||
101 | // Now we can federate the video (reload from database, we need more attributes) | ||
102 | const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t) | ||
103 | await federateVideoIfNeeded(videoForFederation, true, t) | ||
104 | |||
105 | // Update video import object | ||
106 | videoImport.state = VideoImportState.SUCCESS | ||
107 | const videoImportUpdated = await videoImport.save({ transaction: t }) | ||
108 | |||
109 | logger.info('Video %s imported.', videoImport.targetUrl) | ||
110 | |||
111 | videoImportUpdated.Video = videoUpdated | ||
112 | return videoImportUpdated | ||
113 | }) | ||
114 | |||
115 | // Create transcoding jobs? | ||
116 | if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) { | ||
117 | // Put uuid because we don't have id auto incremented for now | ||
118 | const dataInput = { | ||
119 | videoUUID: videoImportUpdated.Video.uuid, | ||
120 | isNewVideo: true | ||
121 | } | ||
122 | |||
123 | await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput }) | ||
124 | } | ||
125 | |||
126 | } catch (err) { | ||
127 | try { | ||
128 | if (tempVideoPath) await unlinkPromise(tempVideoPath) | ||
129 | } catch (errUnlink) { | ||
130 | logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) | ||
131 | } | ||
132 | |||
133 | videoImport.error = err.message | ||
134 | videoImport.state = VideoImportState.FAILED | ||
135 | await videoImport.save() | ||
136 | |||
137 | throw err | ||
138 | } | ||
139 | } | ||
140 | |||
141 | // --------------------------------------------------------------------------- | ||
142 | |||
143 | export { | ||
144 | processVideoImport | ||
145 | } | ||
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 8ff0c169e..8a24604e1 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -2,13 +2,14 @@ import * as Bull from 'bull' | |||
2 | import { JobState, JobType } from '../../../shared/models' | 2 | import { JobState, JobType } from '../../../shared/models' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { Redis } from '../redis' | 4 | import { Redis } from '../redis' |
5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_REQUEST_TTL } from '../../initializers' | 5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers' |
6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' | 6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | 7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' |
8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | 8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' |
9 | import { EmailPayload, processEmail } from './handlers/email' | 9 | import { EmailPayload, processEmail } from './handlers/email' |
10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' | 10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' |
11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' | 11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' | ||
12 | 13 | ||
13 | type CreateJobArgument = | 14 | type CreateJobArgument = |
14 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 15 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -17,7 +18,8 @@ type CreateJobArgument = | |||
17 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | 18 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
18 | { type: 'video-file-import', payload: VideoFileImportPayload } | | 19 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
19 | { type: 'video-file', payload: VideoFilePayload } | | 20 | { type: 'video-file', payload: VideoFilePayload } | |
20 | { type: 'email', payload: EmailPayload } | 21 | { type: 'email', payload: EmailPayload } | |
22 | { type: 'video-import', payload: VideoImportPayload } | ||
21 | 23 | ||
22 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | 24 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { |
23 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, | 25 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
@@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | |||
26 | 'activitypub-follow': processActivityPubFollow, | 28 | 'activitypub-follow': processActivityPubFollow, |
27 | 'video-file-import': processVideoFileImport, | 29 | 'video-file-import': processVideoFileImport, |
28 | 'video-file': processVideoFile, | 30 | 'video-file': processVideoFile, |
29 | 'email': processEmail | 31 | 'email': processEmail, |
32 | 'video-import': processVideoImport | ||
30 | } | 33 | } |
31 | 34 | ||
32 | const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { | 35 | const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { |
@@ -43,7 +46,8 @@ const jobTypes: JobType[] = [ | |||
43 | 'activitypub-http-unicast', | 46 | 'activitypub-http-unicast', |
44 | 'email', | 47 | 'email', |
45 | 'video-file', | 48 | 'video-file', |
46 | 'video-file-import' | 49 | 'video-file-import', |
50 | 'video-import' | ||
47 | ] | 51 | ] |
48 | 52 | ||
49 | class JobQueue { | 53 | class JobQueue { |
@@ -75,7 +79,11 @@ class JobQueue { | |||
75 | const handler = handlers[handlerName] | 79 | const handler = handlers[handlerName] |
76 | 80 | ||
77 | queue.process(JOB_CONCURRENCY[handlerName], handler) | 81 | queue.process(JOB_CONCURRENCY[handlerName], handler) |
78 | .catch(err => logger.error('Cannot execute job queue %s.', handlerName, { err })) | 82 | .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err })) |
83 | |||
84 | queue.on('failed', (job, err) => { | ||
85 | logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err }) | ||
86 | }) | ||
79 | 87 | ||
80 | queue.on('error', err => { | 88 | queue.on('error', err => { |
81 | logger.error('Error in job queue %s.', handlerName, { err }) | 89 | logger.error('Error in job queue %s.', handlerName, { err }) |
@@ -102,11 +110,8 @@ class JobQueue { | |||
102 | 110 | ||
103 | const jobArgs: Bull.JobOptions = { | 111 | const jobArgs: Bull.JobOptions = { |
104 | backoff: { delay: 60 * 1000, type: 'exponential' }, | 112 | backoff: { delay: 60 * 1000, type: 'exponential' }, |
105 | attempts: JOB_ATTEMPTS[obj.type] | 113 | attempts: JOB_ATTEMPTS[obj.type], |
106 | } | 114 | timeout: JOB_TTL[obj.type] |
107 | |||
108 | if (jobsWithRequestTimeout[obj.type] === true) { | ||
109 | jobArgs.timeout = JOB_REQUEST_TTL | ||
110 | } | 115 | } |
111 | 116 | ||
112 | return queue.add(obj.payload, jobArgs) | 117 | return queue.add(obj.payload, jobArgs) |
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts new file mode 100644 index 000000000..a2d919603 --- /dev/null +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts | |||
@@ -0,0 +1,72 @@ | |||
1 | // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js | ||
2 | // We rewrote it to avoid sync calls | ||
3 | |||
4 | import { AbstractScheduler } from './abstract-scheduler' | ||
5 | import { SCHEDULER_INTERVALS_MS } from '../../initializers' | ||
6 | import { logger } from '../../helpers/logger' | ||
7 | import * as request from 'request' | ||
8 | import { createWriteStream, writeFile } from 'fs' | ||
9 | import { join } from 'path' | ||
10 | import { root } from '../../helpers/core-utils' | ||
11 | |||
12 | export class YoutubeDlUpdateScheduler extends AbstractScheduler { | ||
13 | |||
14 | private static instance: AbstractScheduler | ||
15 | |||
16 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.youtubeDLUpdate | ||
17 | |||
18 | private constructor () { | ||
19 | super() | ||
20 | } | ||
21 | |||
22 | async execute () { | ||
23 | const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') | ||
24 | const bin = join(binDirectory, 'youtube-dl') | ||
25 | const detailsPath = join(binDirectory, 'details') | ||
26 | const url = 'https://yt-dl.org/downloads/latest/youtube-dl' | ||
27 | |||
28 | request.get(url, { followRedirect: false }, (err, res) => { | ||
29 | if (err) { | ||
30 | logger.error('Cannot update youtube-dl.', { err }) | ||
31 | return | ||
32 | } | ||
33 | |||
34 | if (res.statusCode !== 302) { | ||
35 | logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', res.statusCode) | ||
36 | return | ||
37 | } | ||
38 | |||
39 | const url = res.headers.location | ||
40 | const downloadFile = request.get(url) | ||
41 | const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1] | ||
42 | |||
43 | downloadFile.on('response', res => { | ||
44 | if (res.statusCode !== 200) { | ||
45 | logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', res.statusCode) | ||
46 | return | ||
47 | } | ||
48 | |||
49 | downloadFile.pipe(createWriteStream(bin, { mode: 493 })) | ||
50 | }) | ||
51 | |||
52 | downloadFile.on('error', err => logger.error('youtube-dl update error.', { err })) | ||
53 | |||
54 | downloadFile.on('end', () => { | ||
55 | const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) | ||
56 | writeFile(detailsPath, details, { encoding: 'utf8' }, err => { | ||
57 | if (err) { | ||
58 | logger.error('youtube-dl update error: cannot write details.', { err }) | ||
59 | return | ||
60 | } | ||
61 | |||
62 | logger.info('youtube-dl updated to version %s.', newVersion) | ||
63 | }) | ||
64 | }) | ||
65 | |||
66 | }) | ||
67 | } | ||
68 | |||
69 | static get Instance () { | ||
70 | return this.instance || (this.instance = new this()) | ||
71 | } | ||
72 | } | ||
diff --git a/server/lib/user.ts b/server/lib/user.ts index ac5f55260..e7a45f5aa 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts | |||
@@ -17,6 +17,7 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse | |||
17 | 17 | ||
18 | const userCreated = await userToCreate.save(userOptions) | 18 | const userCreated = await userToCreate.save(userOptions) |
19 | const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) | 19 | const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) |
20 | userCreated.Account = accountCreated | ||
20 | 21 | ||
21 | const videoChannelDisplayName = `Default ${userCreated.username} channel` | 22 | const videoChannelDisplayName = `Default ${userCreated.username} channel` |
22 | const videoChannelInfo = { | 23 | const videoChannelInfo = { |
diff --git a/server/middlewares/validators/avatar.ts b/server/middlewares/validators/avatar.ts index f346ea92f..5860735c6 100644 --- a/server/middlewares/validators/avatar.ts +++ b/server/middlewares/validators/avatar.ts | |||
@@ -4,6 +4,7 @@ import { isAvatarFile } from '../../helpers/custom-validators/users' | |||
4 | import { areValidationErrors } from './utils' | 4 | import { areValidationErrors } from './utils' |
5 | import { CONSTRAINTS_FIELDS } from '../../initializers' | 5 | import { CONSTRAINTS_FIELDS } from '../../initializers' |
6 | import { logger } from '../../helpers/logger' | 6 | import { logger } from '../../helpers/logger' |
7 | import { cleanUpReqFiles } from '../../helpers/utils' | ||
7 | 8 | ||
8 | const updateAvatarValidator = [ | 9 | const updateAvatarValidator = [ |
9 | body('avatarfile').custom((value, { req }) => isAvatarFile(req.files)).withMessage( | 10 | body('avatarfile').custom((value, { req }) => isAvatarFile(req.files)).withMessage( |
@@ -14,7 +15,7 @@ const updateAvatarValidator = [ | |||
14 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | 15 | (req: express.Request, res: express.Response, next: express.NextFunction) => { |
15 | logger.debug('Checking updateAvatarValidator parameters', { files: req.files }) | 16 | logger.debug('Checking updateAvatarValidator parameters', { files: req.files }) |
16 | 17 | ||
17 | if (areValidationErrors(req, res)) return | 18 | if (areValidationErrors(req, res)) return cleanUpReqFiles(req) |
18 | 19 | ||
19 | return next() | 20 | return next() |
20 | } | 21 | } |
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index f58c0676c..9d303eee2 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts | |||
@@ -24,6 +24,7 @@ const customConfigUpdateValidator = [ | |||
24 | body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), | 24 | body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), |
25 | body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), | 25 | body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), |
26 | body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), | 26 | body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), |
27 | body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), | ||
27 | 28 | ||
28 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 29 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
29 | logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) | 30 | logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) |
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index e3f0f5963..c5400c8f5 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts | |||
@@ -11,3 +11,4 @@ export * from './video-blacklist' | |||
11 | export * from './video-channels' | 11 | export * from './video-channels' |
12 | export * from './webfinger' | 12 | export * from './webfinger' |
13 | export * from './search' | 13 | export * from './search' |
14 | export * from './video-imports' | ||
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index 00bde548c..d85611773 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts | |||
@@ -8,6 +8,7 @@ const SORTABLE_JOBS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.JOBS) | |||
8 | const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES) | 8 | const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES) |
9 | const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS) | 9 | const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS) |
10 | const SORTABLE_VIDEOS_SEARCH_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS_SEARCH) | 10 | const SORTABLE_VIDEOS_SEARCH_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS_SEARCH) |
11 | const SORTABLE_VIDEO_IMPORTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_IMPORTS) | ||
11 | const SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_COMMENT_THREADS) | 12 | const SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_COMMENT_THREADS) |
12 | const SORTABLE_BLACKLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.BLACKLISTS) | 13 | const SORTABLE_BLACKLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.BLACKLISTS) |
13 | const SORTABLE_VIDEO_CHANNELS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_CHANNELS) | 14 | const SORTABLE_VIDEO_CHANNELS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_CHANNELS) |
@@ -19,6 +20,7 @@ const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) | |||
19 | const jobsSortValidator = checkSort(SORTABLE_JOBS_COLUMNS) | 20 | const jobsSortValidator = checkSort(SORTABLE_JOBS_COLUMNS) |
20 | const videoAbusesSortValidator = checkSort(SORTABLE_VIDEO_ABUSES_COLUMNS) | 21 | const videoAbusesSortValidator = checkSort(SORTABLE_VIDEO_ABUSES_COLUMNS) |
21 | const videosSortValidator = checkSort(SORTABLE_VIDEOS_COLUMNS) | 22 | const videosSortValidator = checkSort(SORTABLE_VIDEOS_COLUMNS) |
23 | const videoImportsSortValidator = checkSort(SORTABLE_VIDEO_IMPORTS_COLUMNS) | ||
22 | const videosSearchSortValidator = checkSort(SORTABLE_VIDEOS_SEARCH_COLUMNS) | 24 | const videosSearchSortValidator = checkSort(SORTABLE_VIDEOS_SEARCH_COLUMNS) |
23 | const videoCommentThreadsSortValidator = checkSort(SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS) | 25 | const videoCommentThreadsSortValidator = checkSort(SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS) |
24 | const blacklistSortValidator = checkSort(SORTABLE_BLACKLISTS_COLUMNS) | 26 | const blacklistSortValidator = checkSort(SORTABLE_BLACKLISTS_COLUMNS) |
@@ -32,6 +34,7 @@ export { | |||
32 | usersSortValidator, | 34 | usersSortValidator, |
33 | videoAbusesSortValidator, | 35 | videoAbusesSortValidator, |
34 | videoChannelsSortValidator, | 36 | videoChannelsSortValidator, |
37 | videoImportsSortValidator, | ||
35 | videosSearchSortValidator, | 38 | videosSearchSortValidator, |
36 | videosSortValidator, | 39 | videosSortValidator, |
37 | blacklistSortValidator, | 40 | blacklistSortValidator, |
diff --git a/server/middlewares/validators/video-captions.ts b/server/middlewares/validators/video-captions.ts index b6d92d380..18d537bc4 100644 --- a/server/middlewares/validators/video-captions.ts +++ b/server/middlewares/validators/video-captions.ts | |||
@@ -7,6 +7,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers' | |||
7 | import { UserRight } from '../../../shared' | 7 | import { UserRight } from '../../../shared' |
8 | import { logger } from '../../helpers/logger' | 8 | import { logger } from '../../helpers/logger' |
9 | import { isVideoCaptionExist, isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' | 9 | import { isVideoCaptionExist, isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' |
10 | import { cleanUpReqFiles } from '../../helpers/utils' | ||
10 | 11 | ||
11 | const addVideoCaptionValidator = [ | 12 | const addVideoCaptionValidator = [ |
12 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | 13 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), |
@@ -20,12 +21,12 @@ const addVideoCaptionValidator = [ | |||
20 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 21 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
21 | logger.debug('Checking addVideoCaption parameters', { parameters: req.body }) | 22 | logger.debug('Checking addVideoCaption parameters', { parameters: req.body }) |
22 | 23 | ||
23 | if (areValidationErrors(req, res)) return | 24 | if (areValidationErrors(req, res)) return cleanUpReqFiles(req) |
24 | if (!await isVideoExist(req.params.videoId, res)) return | 25 | if (!await isVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req) |
25 | 26 | ||
26 | // Check if the user who did the request is able to update the video | 27 | // Check if the user who did the request is able to update the video |
27 | const user = res.locals.oauth.token.User | 28 | const user = res.locals.oauth.token.User |
28 | if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return | 29 | if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) |
29 | 30 | ||
30 | return next() | 31 | return next() |
31 | } | 32 | } |
diff --git a/server/middlewares/validators/video-channels.ts b/server/middlewares/validators/video-channels.ts index 7f65f7290..143ce9582 100644 --- a/server/middlewares/validators/video-channels.ts +++ b/server/middlewares/validators/video-channels.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { body, param } from 'express-validator/check' | 2 | import { body, param } from 'express-validator/check' |
3 | import { UserRight } from '../../../shared' | 3 | import { UserRight } from '../../../shared' |
4 | import { isAccountIdExist, isAccountNameWithHostExist } from '../../helpers/custom-validators/accounts' | 4 | import { isAccountNameWithHostExist } from '../../helpers/custom-validators/accounts' |
5 | import { isIdOrUUIDValid } from '../../helpers/custom-validators/misc' | 5 | import { isIdOrUUIDValid } from '../../helpers/custom-validators/misc' |
6 | import { | 6 | import { |
7 | isVideoChannelDescriptionValid, | 7 | isVideoChannelDescriptionValid, |
@@ -13,8 +13,6 @@ import { logger } from '../../helpers/logger' | |||
13 | import { UserModel } from '../../models/account/user' | 13 | import { UserModel } from '../../models/account/user' |
14 | import { VideoChannelModel } from '../../models/video/video-channel' | 14 | import { VideoChannelModel } from '../../models/video/video-channel' |
15 | import { areValidationErrors } from './utils' | 15 | import { areValidationErrors } from './utils' |
16 | import { isAvatarFile } from '../../helpers/custom-validators/users' | ||
17 | import { CONSTRAINTS_FIELDS } from '../../initializers' | ||
18 | 16 | ||
19 | const listVideoAccountChannelsValidator = [ | 17 | const listVideoAccountChannelsValidator = [ |
20 | param('accountName').exists().withMessage('Should have a valid account name'), | 18 | param('accountName').exists().withMessage('Should have a valid account name'), |
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts new file mode 100644 index 000000000..d806edfa3 --- /dev/null +++ b/server/middlewares/validators/video-imports.ts | |||
@@ -0,0 +1,47 @@ | |||
1 | import * as express from 'express' | ||
2 | import { body } from 'express-validator/check' | ||
3 | import { isIdValid } from '../../helpers/custom-validators/misc' | ||
4 | import { logger } from '../../helpers/logger' | ||
5 | import { areValidationErrors } from './utils' | ||
6 | import { getCommonVideoAttributes } from './videos' | ||
7 | import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' | ||
8 | import { cleanUpReqFiles } from '../../helpers/utils' | ||
9 | import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos' | ||
10 | import { CONFIG } from '../../initializers/constants' | ||
11 | |||
12 | const videoImportAddValidator = getCommonVideoAttributes().concat([ | ||
13 | body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'), | ||
14 | body('channelId') | ||
15 | .toInt() | ||
16 | .custom(isIdValid).withMessage('Should have correct video channel id'), | ||
17 | body('name') | ||
18 | .optional() | ||
19 | .custom(isVideoNameValid).withMessage('Should have a valid name'), | ||
20 | |||
21 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
22 | logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body }) | ||
23 | |||
24 | const user = res.locals.oauth.token.User | ||
25 | |||
26 | if (areValidationErrors(req, res)) return cleanUpReqFiles(req) | ||
27 | |||
28 | if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) { | ||
29 | cleanUpReqFiles(req) | ||
30 | return res.status(409) | ||
31 | .json({ error: 'Import is not enabled on this instance.' }) | ||
32 | .end() | ||
33 | } | ||
34 | |||
35 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) | ||
36 | |||
37 | return next() | ||
38 | } | ||
39 | ]) | ||
40 | |||
41 | // --------------------------------------------------------------------------- | ||
42 | |||
43 | export { | ||
44 | videoImportAddValidator | ||
45 | } | ||
46 | |||
47 | // --------------------------------------------------------------------------- | ||
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts index d9af2aa0a..c812d4677 100644 --- a/server/middlewares/validators/videos.ts +++ b/server/middlewares/validators/videos.ts | |||
@@ -35,6 +35,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers' | |||
35 | import { VideoShareModel } from '../../models/video/video-share' | 35 | import { VideoShareModel } from '../../models/video/video-share' |
36 | import { authenticate } from '../oauth' | 36 | import { authenticate } from '../oauth' |
37 | import { areValidationErrors } from './utils' | 37 | import { areValidationErrors } from './utils' |
38 | import { cleanUpReqFiles } from '../../helpers/utils' | ||
38 | 39 | ||
39 | const videosAddValidator = getCommonVideoAttributes().concat([ | 40 | const videosAddValidator = getCommonVideoAttributes().concat([ |
40 | body('videofile') | 41 | body('videofile') |
@@ -50,13 +51,13 @@ const videosAddValidator = getCommonVideoAttributes().concat([ | |||
50 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 51 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
51 | logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) | 52 | logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) |
52 | 53 | ||
53 | if (areValidationErrors(req, res)) return | 54 | if (areValidationErrors(req, res)) return cleanUpReqFiles(req) |
54 | if (areErrorsInScheduleUpdate(req, res)) return | 55 | if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) |
55 | 56 | ||
56 | const videoFile: Express.Multer.File = req.files['videofile'][0] | 57 | const videoFile: Express.Multer.File = req.files['videofile'][0] |
57 | const user = res.locals.oauth.token.User | 58 | const user = res.locals.oauth.token.User |
58 | 59 | ||
59 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return | 60 | if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) |
60 | 61 | ||
61 | const isAble = await user.isAbleToUploadVideo(videoFile) | 62 | const isAble = await user.isAbleToUploadVideo(videoFile) |
62 | if (isAble === false) { | 63 | if (isAble === false) { |
@@ -64,7 +65,7 @@ const videosAddValidator = getCommonVideoAttributes().concat([ | |||
64 | .json({ error: 'The user video quota is exceeded with this video.' }) | 65 | .json({ error: 'The user video quota is exceeded with this video.' }) |
65 | .end() | 66 | .end() |
66 | 67 | ||
67 | return | 68 | return cleanUpReqFiles(req) |
68 | } | 69 | } |
69 | 70 | ||
70 | let duration: number | 71 | let duration: number |
@@ -77,7 +78,7 @@ const videosAddValidator = getCommonVideoAttributes().concat([ | |||
77 | .json({ error: 'Invalid input file.' }) | 78 | .json({ error: 'Invalid input file.' }) |
78 | .end() | 79 | .end() |
79 | 80 | ||
80 | return | 81 | return cleanUpReqFiles(req) |
81 | } | 82 | } |
82 | 83 | ||
83 | videoFile['duration'] = duration | 84 | videoFile['duration'] = duration |
@@ -99,23 +100,24 @@ const videosUpdateValidator = getCommonVideoAttributes().concat([ | |||
99 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 100 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
100 | logger.debug('Checking videosUpdate parameters', { parameters: req.body }) | 101 | logger.debug('Checking videosUpdate parameters', { parameters: req.body }) |
101 | 102 | ||
102 | if (areValidationErrors(req, res)) return | 103 | if (areValidationErrors(req, res)) return cleanUpReqFiles(req) |
103 | if (areErrorsInScheduleUpdate(req, res)) return | 104 | if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) |
104 | if (!await isVideoExist(req.params.id, res)) return | 105 | if (!await isVideoExist(req.params.id, res)) return cleanUpReqFiles(req) |
105 | 106 | ||
106 | const video = res.locals.video | 107 | const video = res.locals.video |
107 | 108 | ||
108 | // Check if the user who did the request is able to update the video | 109 | // Check if the user who did the request is able to update the video |
109 | const user = res.locals.oauth.token.User | 110 | const user = res.locals.oauth.token.User |
110 | if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return | 111 | if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) |
111 | 112 | ||
112 | if (video.privacy !== VideoPrivacy.PRIVATE && req.body.privacy === VideoPrivacy.PRIVATE) { | 113 | if (video.privacy !== VideoPrivacy.PRIVATE && req.body.privacy === VideoPrivacy.PRIVATE) { |
114 | cleanUpReqFiles(req) | ||
113 | return res.status(409) | 115 | return res.status(409) |
114 | .json({ error: 'Cannot set "private" a video that was not private.' }) | 116 | .json({ error: 'Cannot set "private" a video that was not private.' }) |
115 | .end() | 117 | .end() |
116 | } | 118 | } |
117 | 119 | ||
118 | if (req.body.channelId && !await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return | 120 | if (req.body.channelId && !await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) |
119 | 121 | ||
120 | return next() | 122 | return next() |
121 | } | 123 | } |
@@ -221,36 +223,6 @@ const videosShareValidator = [ | |||
221 | } | 223 | } |
222 | ] | 224 | ] |
223 | 225 | ||
224 | // --------------------------------------------------------------------------- | ||
225 | |||
226 | export { | ||
227 | videosAddValidator, | ||
228 | videosUpdateValidator, | ||
229 | videosGetValidator, | ||
230 | videosRemoveValidator, | ||
231 | videosShareValidator, | ||
232 | |||
233 | videoAbuseReportValidator, | ||
234 | |||
235 | videoRateValidator | ||
236 | } | ||
237 | |||
238 | // --------------------------------------------------------------------------- | ||
239 | |||
240 | function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { | ||
241 | if (req.body.scheduleUpdate) { | ||
242 | if (!req.body.scheduleUpdate.updateAt) { | ||
243 | res.status(400) | ||
244 | .json({ error: 'Schedule update at is mandatory.' }) | ||
245 | .end() | ||
246 | |||
247 | return true | ||
248 | } | ||
249 | } | ||
250 | |||
251 | return false | ||
252 | } | ||
253 | |||
254 | function getCommonVideoAttributes () { | 226 | function getCommonVideoAttributes () { |
255 | return [ | 227 | return [ |
256 | body('thumbnailfile') | 228 | body('thumbnailfile') |
@@ -317,3 +289,35 @@ function getCommonVideoAttributes () { | |||
317 | .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy') | 289 | .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy') |
318 | ] as (ValidationChain | express.Handler)[] | 290 | ] as (ValidationChain | express.Handler)[] |
319 | } | 291 | } |
292 | |||
293 | // --------------------------------------------------------------------------- | ||
294 | |||
295 | export { | ||
296 | videosAddValidator, | ||
297 | videosUpdateValidator, | ||
298 | videosGetValidator, | ||
299 | videosRemoveValidator, | ||
300 | videosShareValidator, | ||
301 | |||
302 | videoAbuseReportValidator, | ||
303 | |||
304 | videoRateValidator, | ||
305 | |||
306 | getCommonVideoAttributes | ||
307 | } | ||
308 | |||
309 | // --------------------------------------------------------------------------- | ||
310 | |||
311 | function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { | ||
312 | if (req.body.scheduleUpdate) { | ||
313 | if (!req.body.scheduleUpdate.updateAt) { | ||
314 | res.status(400) | ||
315 | .json({ error: 'Schedule update at is mandatory.' }) | ||
316 | .end() | ||
317 | |||
318 | return true | ||
319 | } | ||
320 | } | ||
321 | |||
322 | return false | ||
323 | } | ||
diff --git a/server/models/account/account.ts b/server/models/account/account.ts index d674d8d22..66f5dcf2e 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts | |||
@@ -16,7 +16,6 @@ import { | |||
16 | } from 'sequelize-typescript' | 16 | } from 'sequelize-typescript' |
17 | import { Account } from '../../../shared/models/actors' | 17 | import { Account } from '../../../shared/models/actors' |
18 | import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' | 18 | import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' |
19 | import { logger } from '../../helpers/logger' | ||
20 | import { sendDeleteActor } from '../../lib/activitypub/send' | 19 | import { sendDeleteActor } from '../../lib/activitypub/send' |
21 | import { ActorModel } from '../activitypub/actor' | 20 | import { ActorModel } from '../activitypub/actor' |
22 | import { ApplicationModel } from '../application/application' | 21 | import { ApplicationModel } from '../application/application' |
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 267032e2a..35d7c35e8 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts | |||
@@ -91,6 +91,9 @@ enum ScopeNames { | |||
91 | fields: [ 'inboxUrl', 'sharedInboxUrl' ] | 91 | fields: [ 'inboxUrl', 'sharedInboxUrl' ] |
92 | }, | 92 | }, |
93 | { | 93 | { |
94 | fields: [ 'sharedInboxUrl' ] | ||
95 | }, | ||
96 | { | ||
94 | fields: [ 'serverId' ] | 97 | fields: [ 'serverId' ] |
95 | }, | 98 | }, |
96 | { | 99 | { |
@@ -454,6 +457,10 @@ export class ActorModel extends Model<ActorModel> { | |||
454 | return 'acct:' + this.preferredUsername + '@' + this.getHost() | 457 | return 'acct:' + this.preferredUsername + '@' + this.getHost() |
455 | } | 458 | } |
456 | 459 | ||
460 | getIdentifier () { | ||
461 | return this.Server ? `${this.preferredUsername}@${this.Server.host}` : this.preferredUsername | ||
462 | } | ||
463 | |||
457 | getHost () { | 464 | getHost () { |
458 | return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST | 465 | return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST |
459 | } | 466 | } |
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts new file mode 100644 index 000000000..eca87163d --- /dev/null +++ b/server/models/video/video-import.ts | |||
@@ -0,0 +1,175 @@ | |||
1 | import { | ||
2 | AfterUpdate, | ||
3 | AllowNull, | ||
4 | BelongsTo, | ||
5 | Column, | ||
6 | CreatedAt, | ||
7 | DataType, | ||
8 | Default, | ||
9 | DefaultScope, | ||
10 | ForeignKey, | ||
11 | Is, | ||
12 | Model, | ||
13 | Table, | ||
14 | UpdatedAt | ||
15 | } from 'sequelize-typescript' | ||
16 | import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' | ||
17 | import { getSort, throwIfNotValid } from '../utils' | ||
18 | import { VideoModel } from './video' | ||
19 | import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' | ||
20 | import { VideoImport, VideoImportState } from '../../../shared' | ||
21 | import { VideoChannelModel } from './video-channel' | ||
22 | import { AccountModel } from '../account/account' | ||
23 | import { TagModel } from './tag' | ||
24 | |||
25 | @DefaultScope({ | ||
26 | include: [ | ||
27 | { | ||
28 | model: () => VideoModel, | ||
29 | required: false, | ||
30 | include: [ | ||
31 | { | ||
32 | model: () => VideoChannelModel, | ||
33 | required: true, | ||
34 | include: [ | ||
35 | { | ||
36 | model: () => AccountModel, | ||
37 | required: true | ||
38 | } | ||
39 | ] | ||
40 | }, | ||
41 | { | ||
42 | model: () => TagModel | ||
43 | } | ||
44 | ] | ||
45 | } | ||
46 | ] | ||
47 | }) | ||
48 | |||
49 | @Table({ | ||
50 | tableName: 'videoImport', | ||
51 | indexes: [ | ||
52 | { | ||
53 | fields: [ 'videoId' ], | ||
54 | unique: true | ||
55 | } | ||
56 | ] | ||
57 | }) | ||
58 | export class VideoImportModel extends Model<VideoImportModel> { | ||
59 | @CreatedAt | ||
60 | createdAt: Date | ||
61 | |||
62 | @UpdatedAt | ||
63 | updatedAt: Date | ||
64 | |||
65 | @AllowNull(false) | ||
66 | @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl')) | ||
67 | @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) | ||
68 | targetUrl: string | ||
69 | |||
70 | @AllowNull(false) | ||
71 | @Default(null) | ||
72 | @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) | ||
73 | @Column | ||
74 | state: VideoImportState | ||
75 | |||
76 | @AllowNull(true) | ||
77 | @Default(null) | ||
78 | @Column(DataType.TEXT) | ||
79 | error: string | ||
80 | |||
81 | @ForeignKey(() => VideoModel) | ||
82 | @Column | ||
83 | videoId: number | ||
84 | |||
85 | @BelongsTo(() => VideoModel, { | ||
86 | foreignKey: { | ||
87 | allowNull: true | ||
88 | }, | ||
89 | onDelete: 'set null' | ||
90 | }) | ||
91 | Video: VideoModel | ||
92 | |||
93 | @AfterUpdate | ||
94 | static deleteVideoIfFailed (instance: VideoImportModel, options) { | ||
95 | if (instance.state === VideoImportState.FAILED) { | ||
96 | return instance.Video.destroy({ transaction: options.transaction }) | ||
97 | } | ||
98 | |||
99 | return undefined | ||
100 | } | ||
101 | |||
102 | static loadAndPopulateVideo (id: number) { | ||
103 | return VideoImportModel.findById(id) | ||
104 | } | ||
105 | |||
106 | static listUserVideoImportsForApi (accountId: number, start: number, count: number, sort: string) { | ||
107 | const query = { | ||
108 | distinct: true, | ||
109 | offset: start, | ||
110 | limit: count, | ||
111 | order: getSort(sort), | ||
112 | include: [ | ||
113 | { | ||
114 | model: VideoModel, | ||
115 | required: false, | ||
116 | include: [ | ||
117 | { | ||
118 | model: VideoChannelModel, | ||
119 | required: true, | ||
120 | include: [ | ||
121 | { | ||
122 | model: AccountModel, | ||
123 | required: true, | ||
124 | where: { | ||
125 | id: accountId | ||
126 | } | ||
127 | } | ||
128 | ] | ||
129 | }, | ||
130 | { | ||
131 | model: TagModel, | ||
132 | required: false | ||
133 | } | ||
134 | ] | ||
135 | } | ||
136 | ] | ||
137 | } | ||
138 | |||
139 | return VideoImportModel.unscoped() | ||
140 | .findAndCountAll(query) | ||
141 | .then(({ rows, count }) => { | ||
142 | return { | ||
143 | data: rows, | ||
144 | total: count | ||
145 | } | ||
146 | }) | ||
147 | } | ||
148 | |||
149 | toFormattedJSON (): VideoImport { | ||
150 | const videoFormatOptions = { | ||
151 | additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true } | ||
152 | } | ||
153 | const video = this.Video | ||
154 | ? Object.assign(this.Video.toFormattedJSON(videoFormatOptions), { | ||
155 | tags: this.Video.Tags.map(t => t.name) | ||
156 | }) | ||
157 | : undefined | ||
158 | |||
159 | return { | ||
160 | id: this.id, | ||
161 | targetUrl: this.targetUrl, | ||
162 | state: { | ||
163 | id: this.state, | ||
164 | label: VideoImportModel.getStateLabel(this.state) | ||
165 | }, | ||
166 | error: this.error, | ||
167 | updatedAt: this.updatedAt.toISOString(), | ||
168 | createdAt: this.createdAt.toISOString(), | ||
169 | video | ||
170 | } | ||
171 | } | ||
172 | private static getStateLabel (id: number) { | ||
173 | return VIDEO_IMPORT_STATES[id] || 'Unknown' | ||
174 | } | ||
175 | } | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index a6c4620b2..39fe21007 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -377,7 +377,7 @@ type AvailableForListOptions = { | |||
377 | include: [ | 377 | include: [ |
378 | { | 378 | { |
379 | model: () => VideoFileModel.unscoped(), | 379 | model: () => VideoFileModel.unscoped(), |
380 | required: true | 380 | required: false |
381 | } | 381 | } |
382 | ] | 382 | ] |
383 | }, | 383 | }, |
@@ -957,8 +957,10 @@ export class VideoModel extends Model<VideoModel> { | |||
957 | }) | 957 | }) |
958 | } | 958 | } |
959 | 959 | ||
960 | static load (id: number) { | 960 | static load (id: number, t?: Sequelize.Transaction) { |
961 | return VideoModel.findById(id) | 961 | const options = t ? { transaction: t } : undefined |
962 | |||
963 | return VideoModel.findById(id, options) | ||
962 | } | 964 | } |
963 | 965 | ||
964 | static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { | 966 | static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { |
@@ -1353,7 +1355,8 @@ export class VideoModel extends Model<VideoModel> { | |||
1353 | mimeType: VIDEO_EXT_MIMETYPE[file.extname], | 1355 | mimeType: VIDEO_EXT_MIMETYPE[file.extname], |
1354 | href: this.getVideoFileUrl(file, baseUrlHttp), | 1356 | href: this.getVideoFileUrl(file, baseUrlHttp), |
1355 | width: file.resolution, | 1357 | width: file.resolution, |
1356 | size: file.size | 1358 | size: file.size, |
1359 | fps: file.fps | ||
1357 | }) | 1360 | }) |
1358 | 1361 | ||
1359 | url.push({ | 1362 | url.push({ |
@@ -1569,21 +1572,25 @@ export class VideoModel extends Model<VideoModel> { | |||
1569 | removeThumbnail () { | 1572 | removeThumbnail () { |
1570 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) | 1573 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) |
1571 | return unlinkPromise(thumbnailPath) | 1574 | return unlinkPromise(thumbnailPath) |
1575 | .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err })) | ||
1572 | } | 1576 | } |
1573 | 1577 | ||
1574 | removePreview () { | 1578 | removePreview () { |
1575 | // Same name than video thumbnail | 1579 | const previewPath = join(CONFIG.STORAGE.PREVIEWS_DIR + this.getPreviewName()) |
1576 | return unlinkPromise(CONFIG.STORAGE.PREVIEWS_DIR + this.getPreviewName()) | 1580 | return unlinkPromise(previewPath) |
1581 | .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err })) | ||
1577 | } | 1582 | } |
1578 | 1583 | ||
1579 | removeFile (videoFile: VideoFileModel) { | 1584 | removeFile (videoFile: VideoFileModel) { |
1580 | const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) | 1585 | const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) |
1581 | return unlinkPromise(filePath) | 1586 | return unlinkPromise(filePath) |
1587 | .catch(err => logger.warn('Cannot delete file %s.', filePath, { err })) | ||
1582 | } | 1588 | } |
1583 | 1589 | ||
1584 | removeTorrent (videoFile: VideoFileModel) { | 1590 | removeTorrent (videoFile: VideoFileModel) { |
1585 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) | 1591 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) |
1586 | return unlinkPromise(torrentPath) | 1592 | return unlinkPromise(torrentPath) |
1593 | .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) | ||
1587 | } | 1594 | } |
1588 | 1595 | ||
1589 | getActivityStreamDuration () { | 1596 | getActivityStreamDuration () { |
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 03855237f..2742e26de 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -60,6 +60,13 @@ describe('Test config API validators', function () { | |||
60 | '720p': false, | 60 | '720p': false, |
61 | '1080p': false | 61 | '1080p': false |
62 | } | 62 | } |
63 | }, | ||
64 | import: { | ||
65 | videos: { | ||
66 | http: { | ||
67 | enabled: false | ||
68 | } | ||
69 | } | ||
63 | } | 70 | } |
64 | } | 71 | } |
65 | 72 | ||
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index 820dde889..03fdd5c4e 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts | |||
@@ -10,4 +10,5 @@ import './video-captions' | |||
10 | import './video-channels' | 10 | import './video-channels' |
11 | import './video-comments' | 11 | import './video-comments' |
12 | import './videos' | 12 | import './videos' |
13 | import './video-imports' | ||
13 | import './search' | 14 | import './search' |
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts new file mode 100644 index 000000000..0ead34a47 --- /dev/null +++ b/server/tests/api/check-params/video-imports.ts | |||
@@ -0,0 +1,275 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import { omit } from 'lodash' | ||
4 | import 'mocha' | ||
5 | import { join } from 'path' | ||
6 | import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' | ||
7 | import { | ||
8 | createUser, | ||
9 | flushTests, | ||
10 | getMyUserInformation, | ||
11 | immutableAssign, | ||
12 | killallServers, | ||
13 | makeGetRequest, | ||
14 | makePostBodyRequest, | ||
15 | makeUploadRequest, | ||
16 | runServer, | ||
17 | ServerInfo, | ||
18 | setAccessTokensToServers, | ||
19 | updateCustomSubConfig, | ||
20 | userLogin | ||
21 | } from '../../utils' | ||
22 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../utils/requests/check-api-params' | ||
23 | import { getYoutubeVideoUrl } from '../../utils/videos/video-imports' | ||
24 | |||
25 | describe('Test video imports API validator', function () { | ||
26 | const path = '/api/v1/videos/imports' | ||
27 | let server: ServerInfo | ||
28 | let userAccessToken = '' | ||
29 | let accountName: string | ||
30 | let channelId: number | ||
31 | let channelUUID: string | ||
32 | |||
33 | // --------------------------------------------------------------- | ||
34 | |||
35 | before(async function () { | ||
36 | this.timeout(30000) | ||
37 | |||
38 | await flushTests() | ||
39 | |||
40 | server = await runServer(1) | ||
41 | |||
42 | await setAccessTokensToServers([ server ]) | ||
43 | |||
44 | const username = 'user1' | ||
45 | const password = 'my super password' | ||
46 | await createUser(server.url, server.accessToken, username, password) | ||
47 | userAccessToken = await userLogin(server, { username, password }) | ||
48 | |||
49 | { | ||
50 | const res = await getMyUserInformation(server.url, server.accessToken) | ||
51 | channelId = res.body.videoChannels[ 0 ].id | ||
52 | channelUUID = res.body.videoChannels[ 0 ].uuid | ||
53 | accountName = res.body.account.name + '@' + res.body.account.host | ||
54 | } | ||
55 | }) | ||
56 | |||
57 | describe('When listing my video imports', function () { | ||
58 | const myPath = '/api/v1/users/me/videos/imports' | ||
59 | |||
60 | it('Should fail with a bad start pagination', async function () { | ||
61 | await checkBadStartPagination(server.url, myPath, server.accessToken) | ||
62 | }) | ||
63 | |||
64 | it('Should fail with a bad count pagination', async function () { | ||
65 | await checkBadCountPagination(server.url, myPath, server.accessToken) | ||
66 | }) | ||
67 | |||
68 | it('Should fail with an incorrect sort', async function () { | ||
69 | await checkBadSortPagination(server.url, myPath, server.accessToken) | ||
70 | }) | ||
71 | |||
72 | it('Should success with the correct parameters', async function () { | ||
73 | await makeGetRequest({ url: server.url, path: myPath, statusCodeExpected: 200, token: server.accessToken }) | ||
74 | }) | ||
75 | }) | ||
76 | |||
77 | describe('When adding a video import', function () { | ||
78 | let baseCorrectParams | ||
79 | |||
80 | before(function () { | ||
81 | baseCorrectParams = { | ||
82 | targetUrl: getYoutubeVideoUrl(), | ||
83 | name: 'my super name', | ||
84 | category: 5, | ||
85 | licence: 1, | ||
86 | language: 'pt', | ||
87 | nsfw: false, | ||
88 | commentsEnabled: true, | ||
89 | waitTranscoding: true, | ||
90 | description: 'my super description', | ||
91 | support: 'my super support text', | ||
92 | tags: [ 'tag1', 'tag2' ], | ||
93 | privacy: VideoPrivacy.PUBLIC, | ||
94 | channelId: channelId | ||
95 | } | ||
96 | }) | ||
97 | |||
98 | it('Should fail with nothing', async function () { | ||
99 | const fields = {} | ||
100 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
101 | }) | ||
102 | |||
103 | it('Should fail without a target url', async function () { | ||
104 | const fields = omit(baseCorrectParams, 'targetUrl') | ||
105 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields, statusCodeExpected: 400 }) | ||
106 | }) | ||
107 | |||
108 | it('Should fail with a bad target url', async function () { | ||
109 | const fields = immutableAssign(baseCorrectParams, { targetUrl: 'htt://hello' }) | ||
110 | |||
111 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
112 | }) | ||
113 | |||
114 | it('Should fail with a long name', async function () { | ||
115 | const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) | ||
116 | |||
117 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
118 | }) | ||
119 | |||
120 | it('Should fail with a bad category', async function () { | ||
121 | const fields = immutableAssign(baseCorrectParams, { category: 125 }) | ||
122 | |||
123 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
124 | }) | ||
125 | |||
126 | it('Should fail with a bad licence', async function () { | ||
127 | const fields = immutableAssign(baseCorrectParams, { licence: 125 }) | ||
128 | |||
129 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
130 | }) | ||
131 | |||
132 | it('Should fail with a bad language', async function () { | ||
133 | const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) | ||
134 | |||
135 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
136 | }) | ||
137 | |||
138 | it('Should fail with a long description', async function () { | ||
139 | const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) | ||
140 | |||
141 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
142 | }) | ||
143 | |||
144 | it('Should fail with a long support text', async function () { | ||
145 | const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(150) }) | ||
146 | |||
147 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
148 | }) | ||
149 | |||
150 | it('Should fail without a channel', async function () { | ||
151 | const fields = omit(baseCorrectParams, 'channelId') | ||
152 | |||
153 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
154 | }) | ||
155 | |||
156 | it('Should fail with a bad channel', async function () { | ||
157 | const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) | ||
158 | |||
159 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
160 | }) | ||
161 | |||
162 | it('Should fail with another user channel', async function () { | ||
163 | const user = { | ||
164 | username: 'fake', | ||
165 | password: 'fake_password' | ||
166 | } | ||
167 | await createUser(server.url, server.accessToken, user.username, user.password) | ||
168 | |||
169 | const accessTokenUser = await userLogin(server, user) | ||
170 | const res = await getMyUserInformation(server.url, accessTokenUser) | ||
171 | const customChannelId = res.body.videoChannels[0].id | ||
172 | |||
173 | const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) | ||
174 | |||
175 | await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields }) | ||
176 | }) | ||
177 | |||
178 | it('Should fail with too many tags', async function () { | ||
179 | const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) | ||
180 | |||
181 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
182 | }) | ||
183 | |||
184 | it('Should fail with a tag length too low', async function () { | ||
185 | const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) | ||
186 | |||
187 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
188 | }) | ||
189 | |||
190 | it('Should fail with a tag length too big', async function () { | ||
191 | const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) | ||
192 | |||
193 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | ||
194 | }) | ||
195 | |||
196 | it('Should fail with an incorrect thumbnail file', async function () { | ||
197 | const fields = baseCorrectParams | ||
198 | const attaches = { | ||
199 | 'thumbnailfile': join(__dirname, '..', '..', 'fixtures', 'avatar.png') | ||
200 | } | ||
201 | |||
202 | await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) | ||
203 | }) | ||
204 | |||
205 | it('Should fail with a big thumbnail file', async function () { | ||
206 | const fields = baseCorrectParams | ||
207 | const attaches = { | ||
208 | 'thumbnailfile': join(__dirname, '..', '..', 'fixtures', 'avatar-big.png') | ||
209 | } | ||
210 | |||
211 | await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) | ||
212 | }) | ||
213 | |||
214 | it('Should fail with an incorrect preview file', async function () { | ||
215 | const fields = baseCorrectParams | ||
216 | const attaches = { | ||
217 | 'previewfile': join(__dirname, '..', '..', 'fixtures', 'avatar.png') | ||
218 | } | ||
219 | |||
220 | await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) | ||
221 | }) | ||
222 | |||
223 | it('Should fail with a big preview file', async function () { | ||
224 | const fields = baseCorrectParams | ||
225 | const attaches = { | ||
226 | 'previewfile': join(__dirname, '..', '..', 'fixtures', 'avatar-big.png') | ||
227 | } | ||
228 | |||
229 | await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) | ||
230 | }) | ||
231 | |||
232 | it('Should succeed with the correct parameters', async function () { | ||
233 | this.timeout(10000) | ||
234 | |||
235 | { | ||
236 | await makePostBodyRequest({ | ||
237 | url: server.url, | ||
238 | path, | ||
239 | token: server.accessToken, | ||
240 | fields: baseCorrectParams, | ||
241 | statusCodeExpected: 200 | ||
242 | }) | ||
243 | } | ||
244 | }) | ||
245 | |||
246 | it('Should forbid to import videos', async function () { | ||
247 | await updateCustomSubConfig(server.url, server.accessToken, { | ||
248 | import: { | ||
249 | videos: { | ||
250 | http: { | ||
251 | enabled: false | ||
252 | } | ||
253 | } | ||
254 | } | ||
255 | }) | ||
256 | |||
257 | await makePostBodyRequest({ | ||
258 | url: server.url, | ||
259 | path, | ||
260 | token: server.accessToken, | ||
261 | fields: baseCorrectParams, | ||
262 | statusCodeExpected: 409 | ||
263 | }) | ||
264 | }) | ||
265 | }) | ||
266 | |||
267 | after(async function () { | ||
268 | killallServers([ server ]) | ||
269 | |||
270 | // Keep the logs if the test failed | ||
271 | if (this['ok']) { | ||
272 | await flushTests() | ||
273 | } | ||
274 | }) | ||
275 | }) | ||
diff --git a/server/tests/api/index-slow.ts b/server/tests/api/index-slow.ts index d987442b3..243c941cb 100644 --- a/server/tests/api/index-slow.ts +++ b/server/tests/api/index-slow.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | // Order of the tests we want to execute | 1 | // Order of the tests we want to execute |
2 | import './videos/video-channels' | ||
2 | import './videos/video-transcoder' | 3 | import './videos/video-transcoder' |
3 | import './videos/multiple-servers' | 4 | import './videos/multiple-servers' |
4 | import './server/follows' | 5 | import './server/follows' |
@@ -7,3 +8,4 @@ import './videos/video-comments' | |||
7 | import './users/users-multiple-servers' | 8 | import './users/users-multiple-servers' |
8 | import './server/handle-down' | 9 | import './server/handle-down' |
9 | import './videos/video-schedule-update' | 10 | import './videos/video-schedule-update' |
11 | import './videos/video-imports' | ||
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 1782a8623..b65061a5d 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -44,6 +44,7 @@ function checkInitialConfig (data: CustomConfig) { | |||
44 | expect(data.transcoding.resolutions['480p']).to.be.true | 44 | expect(data.transcoding.resolutions['480p']).to.be.true |
45 | expect(data.transcoding.resolutions['720p']).to.be.true | 45 | expect(data.transcoding.resolutions['720p']).to.be.true |
46 | expect(data.transcoding.resolutions['1080p']).to.be.true | 46 | expect(data.transcoding.resolutions['1080p']).to.be.true |
47 | expect(data.import.videos.http.enabled).to.be.true | ||
47 | } | 48 | } |
48 | 49 | ||
49 | function checkUpdatedConfig (data: CustomConfig) { | 50 | function checkUpdatedConfig (data: CustomConfig) { |
@@ -70,6 +71,7 @@ function checkUpdatedConfig (data: CustomConfig) { | |||
70 | expect(data.transcoding.resolutions['480p']).to.be.true | 71 | expect(data.transcoding.resolutions['480p']).to.be.true |
71 | expect(data.transcoding.resolutions['720p']).to.be.false | 72 | expect(data.transcoding.resolutions['720p']).to.be.false |
72 | expect(data.transcoding.resolutions['1080p']).to.be.false | 73 | expect(data.transcoding.resolutions['1080p']).to.be.false |
74 | expect(data.import.videos.http.enabled).to.be.false | ||
73 | } | 75 | } |
74 | 76 | ||
75 | describe('Test config', function () { | 77 | describe('Test config', function () { |
@@ -160,6 +162,13 @@ describe('Test config', function () { | |||
160 | '720p': false, | 162 | '720p': false, |
161 | '1080p': false | 163 | '1080p': false |
162 | } | 164 | } |
165 | }, | ||
166 | import: { | ||
167 | videos: { | ||
168 | http: { | ||
169 | enabled: false | ||
170 | } | ||
171 | } | ||
163 | } | 172 | } |
164 | } | 173 | } |
165 | await updateCustomConfig(server.url, server.accessToken, newCustomConfig) | 174 | await updateCustomConfig(server.url, server.accessToken, newCustomConfig) |
diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts index a19b47509..25c87b4dc 100644 --- a/server/tests/api/server/follows.ts +++ b/server/tests/api/server/follows.ts | |||
@@ -178,7 +178,7 @@ describe('Test follows', function () { | |||
178 | }) | 178 | }) |
179 | 179 | ||
180 | it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () { | 180 | it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () { |
181 | this.timeout(10000) | 181 | this.timeout(35000) |
182 | 182 | ||
183 | await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'server2' }) | 183 | await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'server2' }) |
184 | await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3' }) | 184 | await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3' }) |
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index fb9722630..18a0d9ce3 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts | |||
@@ -176,7 +176,7 @@ describe('Test handle downs', function () { | |||
176 | }) | 176 | }) |
177 | 177 | ||
178 | it('Should re-follow server 1', async function () { | 178 | it('Should re-follow server 1', async function () { |
179 | this.timeout(15000) | 179 | this.timeout(35000) |
180 | 180 | ||
181 | await reRunServer(servers[1]) | 181 | await reRunServer(servers[1]) |
182 | await reRunServer(servers[2]) | 182 | await reRunServer(servers[2]) |
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index cb18898ce..516dc5aba 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts | |||
@@ -209,19 +209,19 @@ describe('Test multiple servers', function () { | |||
209 | files: [ | 209 | files: [ |
210 | { | 210 | { |
211 | resolution: 240, | 211 | resolution: 240, |
212 | size: 190000 | 212 | size: 280000 |
213 | }, | 213 | }, |
214 | { | 214 | { |
215 | resolution: 360, | 215 | resolution: 360, |
216 | size: 280000 | 216 | size: 370000 |
217 | }, | 217 | }, |
218 | { | 218 | { |
219 | resolution: 480, | 219 | resolution: 480, |
220 | size: 390000 | 220 | size: 470000 |
221 | }, | 221 | }, |
222 | { | 222 | { |
223 | resolution: 720, | 223 | resolution: 720, |
224 | size: 710000 | 224 | size: 740000 |
225 | } | 225 | } |
226 | ], | 226 | ], |
227 | thumbnailfile: 'thumbnail', | 227 | thumbnailfile: 'thumbnail', |
@@ -975,19 +975,19 @@ describe('Test multiple servers', function () { | |||
975 | files: [ | 975 | files: [ |
976 | { | 976 | { |
977 | resolution: 720, | 977 | resolution: 720, |
978 | size: 40315 | 978 | size: 36000 |
979 | }, | 979 | }, |
980 | { | 980 | { |
981 | resolution: 480, | 981 | resolution: 480, |
982 | size: 22808 | 982 | size: 21000 |
983 | }, | 983 | }, |
984 | { | 984 | { |
985 | resolution: 360, | 985 | resolution: 360, |
986 | size: 18617 | 986 | size: 17000 |
987 | }, | 987 | }, |
988 | { | 988 | { |
989 | resolution: 240, | 989 | resolution: 240, |
990 | size: 15217 | 990 | size: 13000 |
991 | } | 991 | } |
992 | ] | 992 | ] |
993 | } | 993 | } |
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts new file mode 100644 index 000000000..f21ade5c3 --- /dev/null +++ b/server/tests/api/videos/video-imports.ts | |||
@@ -0,0 +1,161 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import * as chai from 'chai' | ||
4 | import 'mocha' | ||
5 | import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos' | ||
6 | import { | ||
7 | doubleFollow, | ||
8 | flushAndRunMultipleServers, | ||
9 | getMyUserInformation, | ||
10 | getMyVideos, | ||
11 | getVideo, | ||
12 | getVideosList, | ||
13 | killallServers, | ||
14 | ServerInfo, | ||
15 | setAccessTokensToServers | ||
16 | } from '../../utils' | ||
17 | import { waitJobs } from '../../utils/server/jobs' | ||
18 | import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../utils/videos/video-imports' | ||
19 | |||
20 | const expect = chai.expect | ||
21 | |||
22 | describe('Test video imports', function () { | ||
23 | let servers: ServerInfo[] = [] | ||
24 | let channelIdServer1: number | ||
25 | let channelIdServer2: number | ||
26 | |||
27 | async function checkVideoServer1 (url: string, id: number | string) { | ||
28 | const res = await getVideo(url, id) | ||
29 | const video: VideoDetails = res.body | ||
30 | |||
31 | expect(video.name).to.equal('small video - youtube') | ||
32 | expect(video.category.label).to.equal('News') | ||
33 | expect(video.licence.label).to.equal('Attribution') | ||
34 | expect(video.language.label).to.equal('Unknown') | ||
35 | expect(video.nsfw).to.be.false | ||
36 | expect(video.description).to.equal('this is a super description') | ||
37 | expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ]) | ||
38 | |||
39 | expect(video.files).to.have.lengthOf(1) | ||
40 | } | ||
41 | |||
42 | async function checkVideoServer2 (url: string, id: number | string) { | ||
43 | const res = await getVideo(url, id) | ||
44 | const video = res.body | ||
45 | |||
46 | expect(video.name).to.equal('my super name') | ||
47 | expect(video.category.label).to.equal('Entertainment') | ||
48 | expect(video.licence.label).to.equal('Public Domain Dedication') | ||
49 | expect(video.language.label).to.equal('English') | ||
50 | expect(video.nsfw).to.be.false | ||
51 | expect(video.description).to.equal('my super description') | ||
52 | expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) | ||
53 | |||
54 | expect(video.files).to.have.lengthOf(1) | ||
55 | } | ||
56 | |||
57 | before(async function () { | ||
58 | this.timeout(30000) | ||
59 | |||
60 | // Run servers | ||
61 | servers = await flushAndRunMultipleServers(2) | ||
62 | |||
63 | await setAccessTokensToServers(servers) | ||
64 | |||
65 | { | ||
66 | const res = await getMyUserInformation(servers[0].url, servers[0].accessToken) | ||
67 | channelIdServer1 = res.body.videoChannels[ 0 ].id | ||
68 | } | ||
69 | |||
70 | { | ||
71 | const res = await getMyUserInformation(servers[1].url, servers[1].accessToken) | ||
72 | channelIdServer2 = res.body.videoChannels[ 0 ].id | ||
73 | } | ||
74 | |||
75 | await doubleFollow(servers[0], servers[1]) | ||
76 | }) | ||
77 | |||
78 | it('Should import a video on server 1', async function () { | ||
79 | this.timeout(60000) | ||
80 | |||
81 | const attributes = { | ||
82 | targetUrl: getYoutubeVideoUrl(), | ||
83 | channelId: channelIdServer1, | ||
84 | privacy: VideoPrivacy.PUBLIC | ||
85 | } | ||
86 | const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) | ||
87 | expect(res.body.video.name).to.equal('small video - youtube') | ||
88 | }) | ||
89 | |||
90 | it('Should list the video to import in my videos on server 1', async function () { | ||
91 | const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5) | ||
92 | |||
93 | expect(res.body.total).to.equal(1) | ||
94 | |||
95 | const videos = res.body.data | ||
96 | expect(videos).to.have.lengthOf(1) | ||
97 | expect(videos[0].name).to.equal('small video - youtube') | ||
98 | }) | ||
99 | |||
100 | it('Should list the video to import in my imports on server 1', async function () { | ||
101 | const res = await getMyVideoImports(servers[0].url, servers[0].accessToken) | ||
102 | |||
103 | expect(res.body.total).to.equal(1) | ||
104 | const videoImports = res.body.data | ||
105 | expect(videoImports).to.have.lengthOf(1) | ||
106 | |||
107 | expect(videoImports[0].targetUrl).to.equal(getYoutubeVideoUrl()) | ||
108 | expect(videoImports[0].video.name).to.equal('small video - youtube') | ||
109 | }) | ||
110 | |||
111 | it('Should have the video listed on the two instances1', async function () { | ||
112 | this.timeout(120000) | ||
113 | |||
114 | await waitJobs(servers) | ||
115 | |||
116 | for (const server of servers) { | ||
117 | const res = await getVideosList(server.url) | ||
118 | expect(res.body.total).to.equal(1) | ||
119 | expect(res.body.data).to.have.lengthOf(1) | ||
120 | |||
121 | await checkVideoServer1(server.url, res.body.data[0].uuid) | ||
122 | } | ||
123 | }) | ||
124 | |||
125 | it('Should import a video on server 2 with some fields', async function () { | ||
126 | this.timeout(60000) | ||
127 | |||
128 | const attributes = { | ||
129 | targetUrl: getYoutubeVideoUrl(), | ||
130 | channelId: channelIdServer1, | ||
131 | privacy: VideoPrivacy.PUBLIC, | ||
132 | category: 10, | ||
133 | licence: 7, | ||
134 | language: 'en', | ||
135 | name: 'my super name', | ||
136 | description: 'my super description', | ||
137 | tags: [ 'supertag1', 'supertag2' ] | ||
138 | } | ||
139 | const res = await importVideo(servers[1].url, servers[1].accessToken, attributes) | ||
140 | expect(res.body.video.name).to.equal('my super name') | ||
141 | }) | ||
142 | |||
143 | it('Should have the video listed on the two instances', async function () { | ||
144 | this.timeout(120000) | ||
145 | |||
146 | await waitJobs(servers) | ||
147 | |||
148 | for (const server of servers) { | ||
149 | const res = await getVideosList(server.url) | ||
150 | expect(res.body.total).to.equal(2) | ||
151 | expect(res.body.data).to.have.lengthOf(2) | ||
152 | |||
153 | await checkVideoServer2(server.url, res.body.data[0].uuid) | ||
154 | await checkVideoServer1(server.url, res.body.data[1].uuid) | ||
155 | } | ||
156 | }) | ||
157 | |||
158 | after(async function () { | ||
159 | killallServers(servers) | ||
160 | }) | ||
161 | }) | ||
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index fe750253e..0f83d4d57 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts | |||
@@ -2,9 +2,12 @@ | |||
2 | 2 | ||
3 | import * as chai from 'chai' | 3 | import * as chai from 'chai' |
4 | import 'mocha' | 4 | import 'mocha' |
5 | import { omit } from 'lodash' | ||
6 | import * as ffmpeg from 'fluent-ffmpeg' | ||
5 | import { VideoDetails, VideoState } from '../../../../shared/models/videos' | 7 | import { VideoDetails, VideoState } from '../../../../shared/models/videos' |
6 | import { getVideoFileFPS } from '../../../helpers/ffmpeg-utils' | 8 | import { getVideoFileFPS, audio } from '../../../helpers/ffmpeg-utils' |
7 | import { | 9 | import { |
10 | buildAbsoluteFixturePath, | ||
8 | doubleFollow, | 11 | doubleFollow, |
9 | flushAndRunMultipleServers, | 12 | flushAndRunMultipleServers, |
10 | getMyVideos, | 13 | getMyVideos, |
@@ -32,6 +35,8 @@ describe('Test video transcoding', function () { | |||
32 | servers = await flushAndRunMultipleServers(2) | 35 | servers = await flushAndRunMultipleServers(2) |
33 | 36 | ||
34 | await setAccessTokensToServers(servers) | 37 | await setAccessTokensToServers(servers) |
38 | |||
39 | await doubleFollow(servers[0], servers[1]) | ||
35 | }) | 40 | }) |
36 | 41 | ||
37 | it('Should not transcode video on server 1', async function () { | 42 | it('Should not transcode video on server 1', async function () { |
@@ -46,20 +51,22 @@ describe('Test video transcoding', function () { | |||
46 | 51 | ||
47 | await waitJobs(servers) | 52 | await waitJobs(servers) |
48 | 53 | ||
49 | const res = await getVideosList(servers[0].url) | 54 | for (const server of servers) { |
50 | const video = res.body.data[0] | 55 | const res = await getVideosList(server.url) |
56 | const video = res.body.data[ 0 ] | ||
51 | 57 | ||
52 | const res2 = await getVideo(servers[0].url, video.id) | 58 | const res2 = await getVideo(server.url, video.id) |
53 | const videoDetails = res2.body | 59 | const videoDetails = res2.body |
54 | expect(videoDetails.files).to.have.lengthOf(1) | 60 | expect(videoDetails.files).to.have.lengthOf(1) |
55 | 61 | ||
56 | const magnetUri = videoDetails.files[0].magnetUri | 62 | const magnetUri = videoDetails.files[ 0 ].magnetUri |
57 | expect(magnetUri).to.match(/\.webm/) | 63 | expect(magnetUri).to.match(/\.webm/) |
58 | 64 | ||
59 | const torrent = await webtorrentAdd(magnetUri) | 65 | const torrent = await webtorrentAdd(magnetUri, true) |
60 | expect(torrent.files).to.be.an('array') | 66 | expect(torrent.files).to.be.an('array') |
61 | expect(torrent.files.length).to.equal(1) | 67 | expect(torrent.files.length).to.equal(1) |
62 | expect(torrent.files[0].path).match(/\.webm$/) | 68 | expect(torrent.files[ 0 ].path).match(/\.webm$/) |
69 | } | ||
63 | }) | 70 | }) |
64 | 71 | ||
65 | it('Should transcode video on server 2', async function () { | 72 | it('Should transcode video on server 2', async function () { |
@@ -74,21 +81,112 @@ describe('Test video transcoding', function () { | |||
74 | 81 | ||
75 | await waitJobs(servers) | 82 | await waitJobs(servers) |
76 | 83 | ||
77 | const res = await getVideosList(servers[1].url) | 84 | for (const server of servers) { |
85 | const res = await getVideosList(server.url) | ||
86 | |||
87 | const video = res.body.data.find(v => v.name === videoAttributes.name) | ||
88 | const res2 = await getVideo(server.url, video.id) | ||
89 | const videoDetails = res2.body | ||
90 | |||
91 | expect(videoDetails.files).to.have.lengthOf(4) | ||
92 | |||
93 | const magnetUri = videoDetails.files[ 0 ].magnetUri | ||
94 | expect(magnetUri).to.match(/\.mp4/) | ||
95 | |||
96 | const torrent = await webtorrentAdd(magnetUri, true) | ||
97 | expect(torrent.files).to.be.an('array') | ||
98 | expect(torrent.files.length).to.equal(1) | ||
99 | expect(torrent.files[ 0 ].path).match(/\.mp4$/) | ||
100 | } | ||
101 | }) | ||
102 | |||
103 | it('Should transcode high bit rate mp3 to proper bit rate', async function () { | ||
104 | this.timeout(60000) | ||
105 | |||
106 | const videoAttributes = { | ||
107 | name: 'mp3_256k', | ||
108 | fixture: 'video_short_mp3_256k.mp4' | ||
109 | } | ||
110 | await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) | ||
111 | |||
112 | await waitJobs(servers) | ||
113 | |||
114 | for (const server of servers) { | ||
115 | const res = await getVideosList(server.url) | ||
116 | |||
117 | const video = res.body.data.find(v => v.name === videoAttributes.name) | ||
118 | const res2 = await getVideo(server.url, video.id) | ||
119 | const videoDetails: VideoDetails = res2.body | ||
120 | |||
121 | expect(videoDetails.files).to.have.lengthOf(4) | ||
122 | |||
123 | const path = join(root(), 'test2', 'videos', video.uuid + '-240.mp4') | ||
124 | const probe = await audio.get(ffmpeg, path) | ||
125 | |||
126 | if (probe.audioStream) { | ||
127 | expect(probe.audioStream[ 'codec_name' ]).to.be.equal('aac') | ||
128 | expect(probe.audioStream[ 'bit_rate' ]).to.be.at.most(384 * 8000) | ||
129 | } else { | ||
130 | this.fail('Could not retrieve the audio stream on ' + probe.absolutePath) | ||
131 | } | ||
132 | } | ||
133 | }) | ||
134 | |||
135 | it('Should transcode video with no audio and have no audio itself', async function () { | ||
136 | this.timeout(60000) | ||
137 | |||
138 | const videoAttributes = { | ||
139 | name: 'no_audio', | ||
140 | fixture: 'video_short_no_audio.mp4' | ||
141 | } | ||
142 | await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) | ||
143 | |||
144 | await waitJobs(servers) | ||
145 | |||
146 | for (const server of servers) { | ||
147 | const res = await getVideosList(server.url) | ||
148 | |||
149 | const video = res.body.data.find(v => v.name === videoAttributes.name) | ||
150 | const res2 = await getVideo(server.url, video.id) | ||
151 | const videoDetails: VideoDetails = res2.body | ||
152 | |||
153 | expect(videoDetails.files).to.have.lengthOf(4) | ||
154 | const path = join(root(), 'test2', 'videos', video.uuid + '-240.mp4') | ||
155 | const probe = await audio.get(ffmpeg, path) | ||
156 | expect(probe).to.not.have.property('audioStream') | ||
157 | } | ||
158 | }) | ||
159 | |||
160 | it('Should leave the audio untouched, but properly transcode the video', async function () { | ||
161 | this.timeout(60000) | ||
162 | |||
163 | const videoAttributes = { | ||
164 | name: 'untouched_audio', | ||
165 | fixture: 'video_short.mp4' | ||
166 | } | ||
167 | await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes) | ||
78 | 168 | ||
79 | const video = res.body.data[0] | 169 | await waitJobs(servers) |
80 | const res2 = await getVideo(servers[1].url, video.id) | ||
81 | const videoDetails = res2.body | ||
82 | 170 | ||
83 | expect(videoDetails.files).to.have.lengthOf(4) | 171 | for (const server of servers) { |
172 | const res = await getVideosList(server.url) | ||
84 | 173 | ||
85 | const magnetUri = videoDetails.files[0].magnetUri | 174 | const video = res.body.data.find(v => v.name === videoAttributes.name) |
86 | expect(magnetUri).to.match(/\.mp4/) | 175 | const res2 = await getVideo(server.url, video.id) |
176 | const videoDetails: VideoDetails = res2.body | ||
87 | 177 | ||
88 | const torrent = await webtorrentAdd(magnetUri) | 178 | expect(videoDetails.files).to.have.lengthOf(4) |
89 | expect(torrent.files).to.be.an('array') | 179 | const fixturePath = buildAbsoluteFixturePath(videoAttributes.fixture) |
90 | expect(torrent.files.length).to.equal(1) | 180 | const fixtureVideoProbe = await audio.get(ffmpeg, fixturePath) |
91 | expect(torrent.files[0].path).match(/\.mp4$/) | 181 | const path = join(root(), 'test2', 'videos', video.uuid + '-240.mp4') |
182 | const videoProbe = await audio.get(ffmpeg, path) | ||
183 | if (videoProbe.audioStream && fixtureVideoProbe.audioStream) { | ||
184 | const toOmit = [ 'max_bit_rate', 'duration', 'duration_ts', 'nb_frames', 'start_time', 'start_pts' ] | ||
185 | expect(omit(videoProbe.audioStream, toOmit)).to.be.deep.equal(omit(fixtureVideoProbe.audioStream, toOmit)) | ||
186 | } else { | ||
187 | this.fail('Could not retrieve the audio stream on ' + videoProbe.absolutePath) | ||
188 | } | ||
189 | } | ||
92 | }) | 190 | }) |
93 | 191 | ||
94 | it('Should transcode a 60 FPS video', async function () { | 192 | it('Should transcode a 60 FPS video', async function () { |
@@ -103,38 +201,36 @@ describe('Test video transcoding', function () { | |||
103 | 201 | ||
104 | await waitJobs(servers) | 202 | await waitJobs(servers) |
105 | 203 | ||
106 | const res = await getVideosList(servers[1].url) | 204 | for (const server of servers) { |
205 | const res = await getVideosList(server.url) | ||
206 | |||
207 | const video = res.body.data.find(v => v.name === videoAttributes.name) | ||
208 | const res2 = await getVideo(server.url, video.id) | ||
209 | const videoDetails: VideoDetails = res2.body | ||
210 | |||
211 | expect(videoDetails.files).to.have.lengthOf(4) | ||
212 | expect(videoDetails.files[ 0 ].fps).to.be.above(58).and.below(62) | ||
213 | expect(videoDetails.files[ 1 ].fps).to.be.below(31) | ||
214 | expect(videoDetails.files[ 2 ].fps).to.be.below(31) | ||
215 | expect(videoDetails.files[ 3 ].fps).to.be.below(31) | ||
107 | 216 | ||
108 | const video = res.body.data[0] | 217 | for (const resolution of [ '240', '360', '480' ]) { |
109 | const res2 = await getVideo(servers[1].url, video.id) | 218 | const path = join(root(), 'test2', 'videos', video.uuid + '-' + resolution + '.mp4') |
110 | const videoDetails: VideoDetails = res2.body | 219 | const fps = await getVideoFileFPS(path) |
111 | 220 | ||
112 | expect(videoDetails.files).to.have.lengthOf(4) | 221 | expect(fps).to.be.below(31) |
113 | expect(videoDetails.files[0].fps).to.be.above(58).and.below(62) | 222 | } |
114 | expect(videoDetails.files[1].fps).to.be.below(31) | ||
115 | expect(videoDetails.files[2].fps).to.be.below(31) | ||
116 | expect(videoDetails.files[3].fps).to.be.below(31) | ||
117 | 223 | ||
118 | for (const resolution of [ '240', '360', '480' ]) { | 224 | const path = join(root(), 'test2', 'videos', video.uuid + '-720.mp4') |
119 | const path = join(root(), 'test2', 'videos', video.uuid + '-' + resolution + '.mp4') | ||
120 | const fps = await getVideoFileFPS(path) | 225 | const fps = await getVideoFileFPS(path) |
121 | 226 | ||
122 | expect(fps).to.be.below(31) | 227 | expect(fps).to.be.above(58).and.below(62) |
123 | } | 228 | } |
124 | |||
125 | const path = join(root(), 'test2', 'videos', video.uuid + '-720.mp4') | ||
126 | const fps = await getVideoFileFPS(path) | ||
127 | |||
128 | expect(fps).to.be.above(58).and.below(62) | ||
129 | }) | 229 | }) |
130 | 230 | ||
131 | it('Should wait transcoding before publishing the video', async function () { | 231 | it('Should wait transcoding before publishing the video', async function () { |
132 | this.timeout(80000) | 232 | this.timeout(80000) |
133 | 233 | ||
134 | await doubleFollow(servers[0], servers[1]) | ||
135 | |||
136 | await waitJobs(servers) | ||
137 | |||
138 | { | 234 | { |
139 | // Upload the video, but wait transcoding | 235 | // Upload the video, but wait transcoding |
140 | const videoAttributes = { | 236 | const videoAttributes = { |
@@ -154,7 +250,7 @@ describe('Test video transcoding', function () { | |||
154 | 250 | ||
155 | // Should have my video | 251 | // Should have my video |
156 | const resMyVideos = await getMyVideos(servers[1].url, servers[1].accessToken, 0, 10) | 252 | const resMyVideos = await getMyVideos(servers[1].url, servers[1].accessToken, 0, 10) |
157 | const videoToFindInMine = resMyVideos.body.data.find(v => v.name === 'waiting video') | 253 | const videoToFindInMine = resMyVideos.body.data.find(v => v.name === videoAttributes.name) |
158 | expect(videoToFindInMine).not.to.be.undefined | 254 | expect(videoToFindInMine).not.to.be.undefined |
159 | expect(videoToFindInMine.state.id).to.equal(VideoState.TO_TRANSCODE) | 255 | expect(videoToFindInMine.state.id).to.equal(VideoState.TO_TRANSCODE) |
160 | expect(videoToFindInMine.state.label).to.equal('To transcode') | 256 | expect(videoToFindInMine.state.label).to.equal('To transcode') |
@@ -162,7 +258,7 @@ describe('Test video transcoding', function () { | |||
162 | 258 | ||
163 | // Should not list this video | 259 | // Should not list this video |
164 | const resVideos = await getVideosList(servers[1].url) | 260 | const resVideos = await getVideosList(servers[1].url) |
165 | const videoToFindInList = resVideos.body.data.find(v => v.name === 'waiting video') | 261 | const videoToFindInList = resVideos.body.data.find(v => v.name === videoAttributes.name) |
166 | expect(videoToFindInList).to.be.undefined | 262 | expect(videoToFindInList).to.be.undefined |
167 | 263 | ||
168 | // Server 1 should not have the video yet | 264 | // Server 1 should not have the video yet |
diff --git a/server/tests/client.ts b/server/tests/client.ts index bcbac86e9..b33a653b1 100644 --- a/server/tests/client.ts +++ b/server/tests/client.ts | |||
@@ -3,17 +3,21 @@ | |||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import * as request from 'supertest' | 5 | import * as request from 'supertest' |
6 | const expect = chai.expect | ||
7 | |||
8 | import { | 6 | import { |
9 | ServerInfo, | ||
10 | flushTests, | 7 | flushTests, |
8 | getCustomConfig, | ||
9 | getVideosList, | ||
10 | killallServers, | ||
11 | makeHTMLRequest, | ||
11 | runServer, | 12 | runServer, |
13 | ServerInfo, | ||
12 | serverLogin, | 14 | serverLogin, |
13 | uploadVideo, | 15 | updateCustomConfig, |
14 | getVideosList, updateCustomConfig, getCustomConfig, killallServers, makeHTMLRequest | 16 | updateCustomSubConfig, |
17 | uploadVideo | ||
15 | } from './utils' | 18 | } from './utils' |
16 | import { CustomConfig } from '../../shared/models/server/custom-config.model' | 19 | |
20 | const expect = chai.expect | ||
17 | 21 | ||
18 | function checkIndexTags (html: string, title: string, description: string, css: string) { | 22 | function checkIndexTags (html: string, title: string, description: string, css: string) { |
19 | expect(html).to.contain('<title>' + title + '</title>') | 23 | expect(html).to.contain('<title>' + title + '</title>') |
@@ -117,56 +121,20 @@ describe('Test a client controllers', function () { | |||
117 | }) | 121 | }) |
118 | 122 | ||
119 | it('Should update the customized configuration and have the correct index html tags', async function () { | 123 | it('Should update the customized configuration and have the correct index html tags', async function () { |
120 | const newCustomConfig: CustomConfig = { | 124 | await updateCustomSubConfig(server.url, server.accessToken, { |
121 | instance: { | 125 | instance: { |
122 | name: 'PeerTube updated', | 126 | name: 'PeerTube updated', |
123 | shortDescription: 'my short description', | 127 | shortDescription: 'my short description', |
124 | description: 'my super description', | 128 | description: 'my super description', |
125 | terms: 'my super terms', | 129 | terms: 'my super terms', |
126 | defaultClientRoute: '/videos/recently-added', | 130 | defaultClientRoute: '/videos/recently-added', |
127 | defaultNSFWPolicy: 'blur' as 'blur', | 131 | defaultNSFWPolicy: 'blur', |
128 | customizations: { | 132 | customizations: { |
129 | javascript: 'alert("coucou")', | 133 | javascript: 'alert("coucou")', |
130 | css: 'body { background-color: red; }' | 134 | css: 'body { background-color: red; }' |
131 | } | 135 | } |
132 | }, | ||
133 | services: { | ||
134 | twitter: { | ||
135 | username: '@Kuja', | ||
136 | whitelisted: true | ||
137 | } | ||
138 | }, | ||
139 | cache: { | ||
140 | previews: { | ||
141 | size: 2 | ||
142 | }, | ||
143 | captions: { | ||
144 | size: 3 | ||
145 | } | ||
146 | }, | ||
147 | signup: { | ||
148 | enabled: false, | ||
149 | limit: 5 | ||
150 | }, | ||
151 | admin: { | ||
152 | email: 'superadmin1@example.com' | ||
153 | }, | ||
154 | user: { | ||
155 | videoQuota: 5242881 | ||
156 | }, | ||
157 | transcoding: { | ||
158 | enabled: true, | ||
159 | threads: 1, | ||
160 | resolutions: { | ||
161 | '240p': false, | ||
162 | '360p': true, | ||
163 | '480p': true, | ||
164 | '720p': false, | ||
165 | '1080p': false | ||
166 | } | ||
167 | } | 136 | } |
168 | } | 137 | }) |
169 | await updateCustomConfig(server.url, server.accessToken, newCustomConfig) | ||
170 | 138 | ||
171 | const res = await makeHTMLRequest(server.url, '/videos/trending') | 139 | const res = await makeHTMLRequest(server.url, '/videos/trending') |
172 | 140 | ||
diff --git a/server/tests/fixtures/video_short_mp3_256k.mp4 b/server/tests/fixtures/video_short_mp3_256k.mp4 new file mode 100644 index 000000000..4c1c7b45e --- /dev/null +++ b/server/tests/fixtures/video_short_mp3_256k.mp4 | |||
Binary files differ | |||
diff --git a/server/tests/fixtures/video_short_no_audio.mp4 b/server/tests/fixtures/video_short_no_audio.mp4 new file mode 100644 index 000000000..329d20fba --- /dev/null +++ b/server/tests/fixtures/video_short_no_audio.mp4 | |||
Binary files differ | |||
diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts index 57f95a603..e21614282 100644 --- a/server/tests/utils/server/config.ts +++ b/server/tests/utils/server/config.ts | |||
@@ -44,6 +44,69 @@ function updateCustomConfig (url: string, token: string, newCustomConfig: Custom | |||
44 | }) | 44 | }) |
45 | } | 45 | } |
46 | 46 | ||
47 | function updateCustomSubConfig (url: string, token: string, newConfig: any) { | ||
48 | const updateParams: CustomConfig = { | ||
49 | instance: { | ||
50 | name: 'PeerTube updated', | ||
51 | shortDescription: 'my short description', | ||
52 | description: 'my super description', | ||
53 | terms: 'my super terms', | ||
54 | defaultClientRoute: '/videos/recently-added', | ||
55 | defaultNSFWPolicy: 'blur', | ||
56 | customizations: { | ||
57 | javascript: 'alert("coucou")', | ||
58 | css: 'body { background-color: red; }' | ||
59 | } | ||
60 | }, | ||
61 | services: { | ||
62 | twitter: { | ||
63 | username: '@MySuperUsername', | ||
64 | whitelisted: true | ||
65 | } | ||
66 | }, | ||
67 | cache: { | ||
68 | previews: { | ||
69 | size: 2 | ||
70 | }, | ||
71 | captions: { | ||
72 | size: 3 | ||
73 | } | ||
74 | }, | ||
75 | signup: { | ||
76 | enabled: false, | ||
77 | limit: 5 | ||
78 | }, | ||
79 | admin: { | ||
80 | email: 'superadmin1@example.com' | ||
81 | }, | ||
82 | user: { | ||
83 | videoQuota: 5242881 | ||
84 | }, | ||
85 | transcoding: { | ||
86 | enabled: true, | ||
87 | threads: 1, | ||
88 | resolutions: { | ||
89 | '240p': false, | ||
90 | '360p': true, | ||
91 | '480p': true, | ||
92 | '720p': false, | ||
93 | '1080p': false | ||
94 | } | ||
95 | }, | ||
96 | import: { | ||
97 | videos: { | ||
98 | http: { | ||
99 | enabled: false | ||
100 | } | ||
101 | } | ||
102 | } | ||
103 | } | ||
104 | |||
105 | Object.assign(updateParams, newConfig) | ||
106 | |||
107 | return updateCustomConfig(url, token, updateParams) | ||
108 | } | ||
109 | |||
47 | function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) { | 110 | function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) { |
48 | const path = '/api/v1/config/custom' | 111 | const path = '/api/v1/config/custom' |
49 | 112 | ||
@@ -62,5 +125,6 @@ export { | |||
62 | getCustomConfig, | 125 | getCustomConfig, |
63 | updateCustomConfig, | 126 | updateCustomConfig, |
64 | getAbout, | 127 | getAbout, |
65 | deleteCustomConfig | 128 | deleteCustomConfig, |
129 | updateCustomSubConfig | ||
66 | } | 130 | } |
diff --git a/server/tests/utils/videos/video-imports.ts b/server/tests/utils/videos/video-imports.ts new file mode 100644 index 000000000..e0f916990 --- /dev/null +++ b/server/tests/utils/videos/video-imports.ts | |||
@@ -0,0 +1,37 @@ | |||
1 | import { VideoImportCreate } from '../../../../shared/models/videos' | ||
2 | import { makeGetRequest, makePostBodyRequest } from '..' | ||
3 | |||
4 | function getYoutubeVideoUrl () { | ||
5 | return 'https://youtu.be/msX3jv1XdvM' | ||
6 | } | ||
7 | |||
8 | function importVideo (url: string, token: string, attributes: VideoImportCreate) { | ||
9 | const path = '/api/v1/videos/imports' | ||
10 | |||
11 | return makePostBodyRequest({ | ||
12 | url, | ||
13 | path, | ||
14 | token, | ||
15 | fields: attributes, | ||
16 | statusCodeExpected: 200 | ||
17 | }) | ||
18 | } | ||
19 | |||
20 | function getMyVideoImports (url: string, token: string) { | ||
21 | const path = '/api/v1/users/me/videos/imports' | ||
22 | |||
23 | return makeGetRequest({ | ||
24 | url, | ||
25 | path, | ||
26 | token, | ||
27 | statusCodeExpected: 200 | ||
28 | }) | ||
29 | } | ||
30 | |||
31 | // --------------------------------------------------------------------------- | ||
32 | |||
33 | export { | ||
34 | getYoutubeVideoUrl, | ||
35 | importVideo, | ||
36 | getMyVideoImports | ||
37 | } | ||
diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts index 8c49eb02b..b280cccda 100644 --- a/server/tests/utils/videos/videos.ts +++ b/server/tests/utils/videos/videos.ts | |||
@@ -522,7 +522,9 @@ async function completeVideoCheck ( | |||
522 | 522 | ||
523 | const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) | 523 | const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) |
524 | const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) | 524 | const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) |
525 | expect(file.size).to.be.above(minSize).and.below(maxSize) | 525 | expect(file.size, |
526 | 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')') | ||
527 | .to.be.above(minSize).and.below(maxSize) | ||
526 | 528 | ||
527 | { | 529 | { |
528 | await testImage(url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) | 530 | await testImage(url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) |