aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/config.ts30
-rw-r--r--server/controllers/api/users.ts78
-rw-r--r--server/controllers/api/video-channel.ts29
-rw-r--r--server/controllers/api/videos/abuse.ts8
-rw-r--r--server/controllers/api/videos/comment.ts10
-rw-r--r--server/controllers/api/videos/import.ts138
-rw-r--r--server/controllers/api/videos/index.ts17
-rw-r--r--server/helpers/activitypub.ts1
-rw-r--r--server/helpers/audit-logger.ts265
-rw-r--r--server/helpers/core-utils.ts2
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts5
-rw-r--r--server/helpers/custom-validators/video-imports.ts49
-rw-r--r--server/helpers/ffmpeg-utils.ts155
-rw-r--r--server/helpers/logger.ts4
-rw-r--r--server/helpers/utils.ts28
-rw-r--r--server/helpers/youtube-dl.ts142
-rw-r--r--server/initializers/checker.ts31
-rw-r--r--server/initializers/constants.ts53
-rw-r--r--server/initializers/database.ts8
-rw-r--r--server/lib/activitypub/process/process-update.ts2
-rw-r--r--server/lib/activitypub/videos.ts3
-rw-r--r--server/lib/job-queue/handlers/video-import.ts145
-rw-r--r--server/lib/job-queue/job-queue.ts25
-rw-r--r--server/lib/schedulers/youtube-dl-update-scheduler.ts72
-rw-r--r--server/lib/user.ts1
-rw-r--r--server/middlewares/validators/avatar.ts3
-rw-r--r--server/middlewares/validators/config.ts1
-rw-r--r--server/middlewares/validators/index.ts1
-rw-r--r--server/middlewares/validators/sort.ts3
-rw-r--r--server/middlewares/validators/video-captions.ts7
-rw-r--r--server/middlewares/validators/video-channels.ts4
-rw-r--r--server/middlewares/validators/video-imports.ts47
-rw-r--r--server/middlewares/validators/videos.ts84
-rw-r--r--server/models/account/account.ts1
-rw-r--r--server/models/activitypub/actor.ts7
-rw-r--r--server/models/video/video-import.ts175
-rw-r--r--server/models/video/video.ts19
-rw-r--r--server/tests/api/check-params/config.ts7
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/video-imports.ts275
-rw-r--r--server/tests/api/index-slow.ts2
-rw-r--r--server/tests/api/server/config.ts9
-rw-r--r--server/tests/api/server/follows.ts2
-rw-r--r--server/tests/api/server/handle-down.ts2
-rw-r--r--server/tests/api/videos/multiple-servers.ts16
-rw-r--r--server/tests/api/videos/video-imports.ts161
-rw-r--r--server/tests/api/videos/video-transcoder.ts188
-rw-r--r--server/tests/client.ts58
-rw-r--r--server/tests/fixtures/video_short_mp3_256k.mp4bin0 -> 194985 bytes
-rw-r--r--server/tests/fixtures/video_short_no_audio.mp4bin0 -> 34259 bytes
-rw-r--r--server/tests/utils/server/config.ts66
-rw-r--r--server/tests/utils/videos/video-imports.ts37
-rw-r--r--server/tests/utils/videos/videos.ts4
53 files changed, 2267 insertions, 214 deletions
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 9c1b2818c..950a1498e 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -9,10 +9,13 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers'
9import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' 9import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
10import { customConfigUpdateValidator } from '../../middlewares/validators/config' 10import { customConfigUpdateValidator } from '../../middlewares/validators/config'
11import { ClientHtml } from '../../lib/client-html' 11import { ClientHtml } from '../../lib/client-html'
12import { CustomConfigAuditView, auditLoggerFactory } from '../../helpers/audit-logger'
12 13
13const packageJSON = require('../../../../package.json') 14const packageJSON = require('../../../../package.json')
14const configRouter = express.Router() 15const configRouter = express.Router()
15 16
17const auditLogger = auditLoggerFactory('config')
18
16configRouter.get('/about', getAbout) 19configRouter.get('/about', getAbout)
17configRouter.get('/', 20configRouter.get('/',
18 asyncMiddleware(getConfig) 21 asyncMiddleware(getConfig)
@@ -62,6 +65,13 @@ async function getConfig (req: express.Request, res: express.Response, next: exp
62 transcoding: { 65 transcoding: {
63 enabledResolutions 66 enabledResolutions
64 }, 67 },
68 import: {
69 videos: {
70 http: {
71 enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
72 }
73 }
74 },
65 avatar: { 75 avatar: {
66 file: { 76 file: {
67 size: { 77 size: {
@@ -119,6 +129,11 @@ async function getCustomConfig (req: express.Request, res: express.Response, nex
119async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { 129async function deleteCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) {
120 await unlinkPromise(CONFIG.CUSTOM_FILE) 130 await unlinkPromise(CONFIG.CUSTOM_FILE)
121 131
132 auditLogger.delete(
133 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
134 new CustomConfigAuditView(customConfig())
135 )
136
122 reloadConfig() 137 reloadConfig()
123 ClientHtml.invalidCache() 138 ClientHtml.invalidCache()
124 139
@@ -129,6 +144,7 @@ async function deleteCustomConfig (req: express.Request, res: express.Response,
129 144
130async function updateCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) { 145async function updateCustomConfig (req: express.Request, res: express.Response, next: express.NextFunction) {
131 const toUpdate: CustomConfig = req.body 146 const toUpdate: CustomConfig = req.body
147 const oldCustomConfigAuditKeys = new CustomConfigAuditView(customConfig())
132 148
133 // Force number conversion 149 // Force number conversion
134 toUpdate.cache.previews.size = parseInt('' + toUpdate.cache.previews.size, 10) 150 toUpdate.cache.previews.size = parseInt('' + toUpdate.cache.previews.size, 10)
@@ -150,6 +166,13 @@ async function updateCustomConfig (req: express.Request, res: express.Response,
150 ClientHtml.invalidCache() 166 ClientHtml.invalidCache()
151 167
152 const data = customConfig() 168 const data = customConfig()
169
170 auditLogger.update(
171 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
172 new CustomConfigAuditView(data),
173 oldCustomConfigAuditKeys
174 )
175
153 return res.json(data).end() 176 return res.json(data).end()
154} 177}
155 178
@@ -209,6 +232,13 @@ function customConfig (): CustomConfig {
209 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], 232 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ],
210 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] 233 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ]
211 } 234 }
235 },
236 import: {
237 videos: {
238 http: {
239 enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
240 }
241 }
212 } 242 }
213 } 243 }
214} 244}
diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts
index c80f27a23..879ba3f91 100644
--- a/server/controllers/api/users.ts
+++ b/server/controllers/api/users.ts
@@ -29,7 +29,12 @@ import {
29 usersUpdateValidator, 29 usersUpdateValidator,
30 usersVideoRatingValidator 30 usersVideoRatingValidator
31} from '../../middlewares' 31} from '../../middlewares'
32import { usersAskResetPasswordValidator, usersResetPasswordValidator, videosSortValidator } from '../../middlewares/validators' 32import {
33 usersAskResetPasswordValidator,
34 usersResetPasswordValidator,
35 videoImportsSortValidator,
36 videosSortValidator
37} from '../../middlewares/validators'
33import { AccountVideoRateModel } from '../../models/account/account-video-rate' 38import { AccountVideoRateModel } from '../../models/account/account-video-rate'
34import { UserModel } from '../../models/account/user' 39import { UserModel } from '../../models/account/user'
35import { OAuthTokenModel } from '../../models/oauth/oauth-token' 40import { OAuthTokenModel } from '../../models/oauth/oauth-token'
@@ -39,6 +44,10 @@ import { createReqFiles } from '../../helpers/express-utils'
39import { UserVideoQuota } from '../../../shared/models/users/user-video-quota.model' 44import { UserVideoQuota } from '../../../shared/models/users/user-video-quota.model'
40import { updateAvatarValidator } from '../../middlewares/validators/avatar' 45import { updateAvatarValidator } from '../../middlewares/validators/avatar'
41import { updateActorAvatarFile } from '../../lib/avatar' 46import { updateActorAvatarFile } from '../../lib/avatar'
47import { auditLoggerFactory, UserAuditView } from '../../helpers/audit-logger'
48import { VideoImportModel } from '../../models/video/video-import'
49
50const auditLogger = auditLoggerFactory('users')
42 51
43const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) 52const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
44const loginRateLimiter = new RateLimit({ 53const loginRateLimiter = new RateLimit({
@@ -59,6 +68,15 @@ usersRouter.get('/me/video-quota-used',
59 asyncMiddleware(getUserVideoQuotaUsed) 68 asyncMiddleware(getUserVideoQuotaUsed)
60) 69)
61 70
71usersRouter.get('/me/videos/imports',
72 authenticate,
73 paginationValidator,
74 videoImportsSortValidator,
75 setDefaultSort,
76 setDefaultPagination,
77 asyncMiddleware(getUserVideoImports)
78)
79
62usersRouter.get('/me/videos', 80usersRouter.get('/me/videos',
63 authenticate, 81 authenticate,
64 paginationValidator, 82 paginationValidator,
@@ -175,6 +193,18 @@ async function getUserVideos (req: express.Request, res: express.Response, next:
175 return res.json(getFormattedObjects(resultList.data, resultList.total, { additionalAttributes })) 193 return res.json(getFormattedObjects(resultList.data, resultList.total, { additionalAttributes }))
176} 194}
177 195
196async function getUserVideoImports (req: express.Request, res: express.Response, next: express.NextFunction) {
197 const user = res.locals.oauth.token.User as UserModel
198 const resultList = await VideoImportModel.listUserVideoImportsForApi(
199 user.Account.id,
200 req.query.start as number,
201 req.query.count as number,
202 req.query.sort
203 )
204
205 return res.json(getFormattedObjects(resultList.data, resultList.total))
206}
207
178async function createUser (req: express.Request, res: express.Response) { 208async function createUser (req: express.Request, res: express.Response) {
179 const body: UserCreate = req.body 209 const body: UserCreate = req.body
180 const userToCreate = new UserModel({ 210 const userToCreate = new UserModel({
@@ -189,6 +219,7 @@ async function createUser (req: express.Request, res: express.Response) {
189 219
190 const { user, account } = await createUserAccountAndChannel(userToCreate) 220 const { user, account } = await createUserAccountAndChannel(userToCreate)
191 221
222 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON()))
192 logger.info('User %s with its channel and account created.', body.username) 223 logger.info('User %s with its channel and account created.', body.username)
193 224
194 return res.json({ 225 return res.json({
@@ -205,7 +236,7 @@ async function createUser (req: express.Request, res: express.Response) {
205async function registerUser (req: express.Request, res: express.Response) { 236async function registerUser (req: express.Request, res: express.Response) {
206 const body: UserCreate = req.body 237 const body: UserCreate = req.body
207 238
208 const user = new UserModel({ 239 const userToCreate = new UserModel({
209 username: body.username, 240 username: body.username,
210 password: body.password, 241 password: body.password,
211 email: body.email, 242 email: body.email,
@@ -215,8 +246,9 @@ async function registerUser (req: express.Request, res: express.Response) {
215 videoQuota: CONFIG.USER.VIDEO_QUOTA 246 videoQuota: CONFIG.USER.VIDEO_QUOTA
216 }) 247 })
217 248
218 await createUserAccountAndChannel(user) 249 const { user } = await createUserAccountAndChannel(userToCreate)
219 250
251 auditLogger.create(body.username, new UserAuditView(user.toFormattedJSON()))
220 logger.info('User %s with its channel and account registered.', body.username) 252 logger.info('User %s with its channel and account registered.', body.username)
221 253
222 return res.type('json').status(204).end() 254 return res.type('json').status(204).end()
@@ -269,6 +301,8 @@ async function removeUser (req: express.Request, res: express.Response, next: ex
269 301
270 await user.destroy() 302 await user.destroy()
271 303
304 auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new UserAuditView(user.toFormattedJSON()))
305
272 return res.sendStatus(204) 306 return res.sendStatus(204)
273} 307}
274 308
@@ -276,6 +310,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
276 const body: UserUpdateMe = req.body 310 const body: UserUpdateMe = req.body
277 311
278 const user: UserModel = res.locals.oauth.token.user 312 const user: UserModel = res.locals.oauth.token.user
313 const oldUserAuditView = new UserAuditView(user.toFormattedJSON())
279 314
280 if (body.password !== undefined) user.password = body.password 315 if (body.password !== undefined) user.password = body.password
281 if (body.email !== undefined) user.email = body.email 316 if (body.email !== undefined) user.email = body.email
@@ -290,6 +325,12 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
290 await user.Account.save({ transaction: t }) 325 await user.Account.save({ transaction: t })
291 326
292 await sendUpdateActor(user.Account, t) 327 await sendUpdateActor(user.Account, t)
328
329 auditLogger.update(
330 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
331 new UserAuditView(user.toFormattedJSON()),
332 oldUserAuditView
333 )
293 }) 334 })
294 335
295 return res.sendStatus(204) 336 return res.sendStatus(204)
@@ -297,10 +338,18 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
297 338
298async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { 339async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) {
299 const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] 340 const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
300 const account = res.locals.oauth.token.user.Account 341 const user: UserModel = res.locals.oauth.token.user
342 const oldUserAuditView = new UserAuditView(user.toFormattedJSON())
343 const account = user.Account
301 344
302 const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account) 345 const avatar = await updateActorAvatarFile(avatarPhysicalFile, account.Actor, account)
303 346
347 auditLogger.update(
348 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
349 new UserAuditView(user.toFormattedJSON()),
350 oldUserAuditView
351 )
352
304 return res 353 return res
305 .json({ 354 .json({
306 avatar: avatar.toFormattedJSON() 355 avatar: avatar.toFormattedJSON()
@@ -310,20 +359,27 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next
310 359
311async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) { 360async function updateUser (req: express.Request, res: express.Response, next: express.NextFunction) {
312 const body: UserUpdate = req.body 361 const body: UserUpdate = req.body
313 const user = res.locals.user as UserModel 362 const userToUpdate = res.locals.user as UserModel
314 const roleChanged = body.role !== undefined && body.role !== user.role 363 const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON())
364 const roleChanged = body.role !== undefined && body.role !== userToUpdate.role
315 365
316 if (body.email !== undefined) user.email = body.email 366 if (body.email !== undefined) userToUpdate.email = body.email
317 if (body.videoQuota !== undefined) user.videoQuota = body.videoQuota 367 if (body.videoQuota !== undefined) userToUpdate.videoQuota = body.videoQuota
318 if (body.role !== undefined) user.role = body.role 368 if (body.role !== undefined) userToUpdate.role = body.role
319 369
320 await user.save() 370 const user = await userToUpdate.save()
321 371
322 // Destroy user token to refresh rights 372 // Destroy user token to refresh rights
323 if (roleChanged) { 373 if (roleChanged) {
324 await OAuthTokenModel.deleteUserToken(user.id) 374 await OAuthTokenModel.deleteUserToken(userToUpdate.id)
325 } 375 }
326 376
377 auditLogger.update(
378 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
379 new UserAuditView(user.toFormattedJSON()),
380 oldUserAuditView
381 )
382
327 // Don't need to send this update to followers, these attributes are not propagated 383 // Don't need to send this update to followers, these attributes are not propagated
328 384
329 return res.sendStatus(204) 385 return res.sendStatus(204)
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts
index 0488ba8f5..3a444547b 100644
--- a/server/controllers/api/video-channel.ts
+++ b/server/controllers/api/video-channel.ts
@@ -27,7 +27,9 @@ import { logger } from '../../helpers/logger'
27import { VideoModel } from '../../models/video/video' 27import { VideoModel } from '../../models/video/video'
28import { updateAvatarValidator } from '../../middlewares/validators/avatar' 28import { updateAvatarValidator } from '../../middlewares/validators/avatar'
29import { updateActorAvatarFile } from '../../lib/avatar' 29import { updateActorAvatarFile } from '../../lib/avatar'
30import { auditLoggerFactory, VideoChannelAuditView } from '../../helpers/audit-logger'
30 31
32const auditLogger = auditLoggerFactory('channels')
31const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR }) 33const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
32 34
33const videoChannelRouter = express.Router() 35const videoChannelRouter = express.Router()
@@ -99,10 +101,17 @@ async function listVideoChannels (req: express.Request, res: express.Response, n
99 101
100async function updateVideoChannelAvatar (req: express.Request, res: express.Response, next: express.NextFunction) { 102async function updateVideoChannelAvatar (req: express.Request, res: express.Response, next: express.NextFunction) {
101 const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ] 103 const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
102 const videoChannel = res.locals.videoChannel 104 const videoChannel = res.locals.videoChannel as VideoChannelModel
105 const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
103 106
104 const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel) 107 const avatar = await updateActorAvatarFile(avatarPhysicalFile, videoChannel.Actor, videoChannel)
105 108
109 auditLogger.update(
110 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
111 new VideoChannelAuditView(videoChannel.toFormattedJSON()),
112 oldVideoChannelAuditKeys
113 )
114
106 return res 115 return res
107 .json({ 116 .json({
108 avatar: avatar.toFormattedJSON() 117 avatar: avatar.toFormattedJSON()
@@ -121,6 +130,10 @@ async function addVideoChannel (req: express.Request, res: express.Response) {
121 setAsyncActorKeys(videoChannelCreated.Actor) 130 setAsyncActorKeys(videoChannelCreated.Actor)
122 .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err })) 131 .catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.uuid, { err }))
123 132
133 auditLogger.create(
134 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
135 new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())
136 )
124 logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid) 137 logger.info('Video channel with uuid %s created.', videoChannelCreated.Actor.uuid)
125 138
126 return res.json({ 139 return res.json({
@@ -134,6 +147,7 @@ async function addVideoChannel (req: express.Request, res: express.Response) {
134async function updateVideoChannel (req: express.Request, res: express.Response) { 147async function updateVideoChannel (req: express.Request, res: express.Response) {
135 const videoChannelInstance = res.locals.videoChannel as VideoChannelModel 148 const videoChannelInstance = res.locals.videoChannel as VideoChannelModel
136 const videoChannelFieldsSave = videoChannelInstance.toJSON() 149 const videoChannelFieldsSave = videoChannelInstance.toJSON()
150 const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())
137 const videoChannelInfoToUpdate = req.body as VideoChannelUpdate 151 const videoChannelInfoToUpdate = req.body as VideoChannelUpdate
138 152
139 try { 153 try {
@@ -148,9 +162,14 @@ async function updateVideoChannel (req: express.Request, res: express.Response)
148 162
149 const videoChannelInstanceUpdated = await videoChannelInstance.save(sequelizeOptions) 163 const videoChannelInstanceUpdated = await videoChannelInstance.save(sequelizeOptions)
150 await sendUpdateActor(videoChannelInstanceUpdated, t) 164 await sendUpdateActor(videoChannelInstanceUpdated, t)
151 })
152 165
153 logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) 166 auditLogger.update(
167 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
168 new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()),
169 oldVideoChannelAuditKeys
170 )
171 logger.info('Video channel with name %s and uuid %s updated.', videoChannelInstance.name, videoChannelInstance.Actor.uuid)
172 })
154 } catch (err) { 173 } catch (err) {
155 logger.debug('Cannot update the video channel.', { err }) 174 logger.debug('Cannot update the video channel.', { err })
156 175
@@ -171,6 +190,10 @@ async function removeVideoChannel (req: express.Request, res: express.Response)
171 await sequelizeTypescript.transaction(async t => { 190 await sequelizeTypescript.transaction(async t => {
172 await videoChannelInstance.destroy({ transaction: t }) 191 await videoChannelInstance.destroy({ transaction: t })
173 192
193 auditLogger.delete(
194 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
195 new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())
196 )
174 logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid) 197 logger.info('Video channel with name %s and uuid %s deleted.', videoChannelInstance.name, videoChannelInstance.Actor.uuid)
175 }) 198 })
176 199
diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts
index 3413ae894..7782fc639 100644
--- a/server/controllers/api/videos/abuse.ts
+++ b/server/controllers/api/videos/abuse.ts
@@ -18,7 +18,9 @@ import {
18import { AccountModel } from '../../../models/account/account' 18import { AccountModel } from '../../../models/account/account'
19import { VideoModel } from '../../../models/video/video' 19import { VideoModel } from '../../../models/video/video'
20import { VideoAbuseModel } from '../../../models/video/video-abuse' 20import { VideoAbuseModel } from '../../../models/video/video-abuse'
21import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-logger'
21 22
23const auditLogger = auditLoggerFactory('abuse')
22const abuseVideoRouter = express.Router() 24const abuseVideoRouter = express.Router()
23 25
24abuseVideoRouter.get('/abuse', 26abuseVideoRouter.get('/abuse',
@@ -64,14 +66,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
64 await sequelizeTypescript.transaction(async t => { 66 await sequelizeTypescript.transaction(async t => {
65 const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t }) 67 const videoAbuseInstance = await VideoAbuseModel.create(abuseToCreate, { transaction: t })
66 videoAbuseInstance.Video = videoInstance 68 videoAbuseInstance.Video = videoInstance
69 videoAbuseInstance.Account = reporterAccount
67 70
68 // We send the video abuse to the origin server 71 // We send the video abuse to the origin server
69 if (videoInstance.isOwned() === false) { 72 if (videoInstance.isOwned() === false) {
70 await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) 73 await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
71 } 74 }
72 })
73 75
74 logger.info('Abuse report for video %s created.', videoInstance.name) 76 auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
77 logger.info('Abuse report for video %s created.', videoInstance.name)
78 })
75 79
76 return res.type('json').status(204).end() 80 return res.type('json').status(204).end()
77} 81}
diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts
index bbeb0d557..e35247829 100644
--- a/server/controllers/api/videos/comment.ts
+++ b/server/controllers/api/videos/comment.ts
@@ -23,7 +23,9 @@ import {
23} from '../../../middlewares/validators/video-comments' 23} from '../../../middlewares/validators/video-comments'
24import { VideoModel } from '../../../models/video/video' 24import { VideoModel } from '../../../models/video/video'
25import { VideoCommentModel } from '../../../models/video/video-comment' 25import { VideoCommentModel } from '../../../models/video/video-comment'
26import { auditLoggerFactory, CommentAuditView } from '../../../helpers/audit-logger'
26 27
28const auditLogger = auditLoggerFactory('comments')
27const videoCommentRouter = express.Router() 29const videoCommentRouter = express.Router()
28 30
29videoCommentRouter.get('/:videoId/comment-threads', 31videoCommentRouter.get('/:videoId/comment-threads',
@@ -107,6 +109,8 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons
107 }, t) 109 }, t)
108 }) 110 })
109 111
112 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON()))
113
110 return res.json({ 114 return res.json({
111 comment: comment.toFormattedJSON() 115 comment: comment.toFormattedJSON()
112 }).end() 116 }).end()
@@ -124,6 +128,8 @@ async function addVideoCommentReply (req: express.Request, res: express.Response
124 }, t) 128 }, t)
125 }) 129 })
126 130
131 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new CommentAuditView(comment.toFormattedJSON()))
132
127 return res.json({ 133 return res.json({
128 comment: comment.toFormattedJSON() 134 comment: comment.toFormattedJSON()
129 }).end() 135 }).end()
@@ -136,6 +142,10 @@ async function removeVideoComment (req: express.Request, res: express.Response)
136 await videoCommentInstance.destroy({ transaction: t }) 142 await videoCommentInstance.destroy({ transaction: t })
137 }) 143 })
138 144
145 auditLogger.delete(
146 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
147 new CommentAuditView(videoCommentInstance.toFormattedJSON())
148 )
139 logger.info('Video comment %d deleted.', videoCommentInstance.id) 149 logger.info('Video comment %d deleted.', videoCommentInstance.id)
140 150
141 return res.type('json').status(204).end() 151 return res.type('json').status(204).end()
diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
new file mode 100644
index 000000000..30a7d816c
--- /dev/null
+++ b/server/controllers/api/videos/import.ts
@@ -0,0 +1,138 @@
1import * as express from 'express'
2import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
3import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
4import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
5import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
6import { createReqFiles } from '../../../helpers/express-utils'
7import { logger } from '../../../helpers/logger'
8import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
9import { VideoModel } from '../../../models/video/video'
10import { getVideoActivityPubUrl } from '../../../lib/activitypub'
11import { TagModel } from '../../../models/video/tag'
12import { VideoImportModel } from '../../../models/video/video-import'
13import { JobQueue } from '../../../lib/job-queue/job-queue'
14import { processImage } from '../../../helpers/image-utils'
15import { join } from 'path'
16
17const auditLogger = auditLoggerFactory('video-imports')
18const videoImportsRouter = express.Router()
19
20const reqVideoFileImport = createReqFiles(
21 [ 'thumbnailfile', 'previewfile' ],
22 IMAGE_MIMETYPE_EXT,
23 {
24 thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
25 previewfile: CONFIG.STORAGE.PREVIEWS_DIR
26 }
27)
28
29videoImportsRouter.post('/imports',
30 authenticate,
31 reqVideoFileImport,
32 asyncMiddleware(videoImportAddValidator),
33 asyncRetryTransactionMiddleware(addVideoImport)
34)
35
36// ---------------------------------------------------------------------------
37
38export {
39 videoImportsRouter
40}
41
42// ---------------------------------------------------------------------------
43
44async function addVideoImport (req: express.Request, res: express.Response) {
45 const body: VideoImportCreate = req.body
46 const targetUrl = body.targetUrl
47
48 let youtubeDLInfo: YoutubeDLInfo
49 try {
50 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
51 } catch (err) {
52 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
53
54 return res.status(400).json({
55 error: 'Cannot fetch remote information of this URL.'
56 }).end()
57 }
58
59 // Create video DB object
60 const videoData = {
61 name: body.name || youtubeDLInfo.name,
62 remote: false,
63 category: body.category || youtubeDLInfo.category,
64 licence: body.licence || youtubeDLInfo.licence,
65 language: body.language || undefined,
66 commentsEnabled: body.commentsEnabled || true,
67 waitTranscoding: body.waitTranscoding || false,
68 state: VideoState.TO_IMPORT,
69 nsfw: body.nsfw || youtubeDLInfo.nsfw || false,
70 description: body.description || youtubeDLInfo.description,
71 support: body.support || null,
72 privacy: body.privacy || VideoPrivacy.PRIVATE,
73 duration: 0, // duration will be set by the import job
74 channelId: res.locals.videoChannel.id
75 }
76 const video = new VideoModel(videoData)
77 video.url = getVideoActivityPubUrl(video)
78
79 // Process thumbnail file?
80 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
81 let downloadThumbnail = true
82 if (thumbnailField) {
83 const thumbnailPhysicalFile = thumbnailField[ 0 ]
84 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
85 downloadThumbnail = false
86 }
87
88 // Process preview file?
89 const previewField = req.files ? req.files['previewfile'] : undefined
90 let downloadPreview = true
91 if (previewField) {
92 const previewPhysicalFile = previewField[0]
93 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
94 downloadPreview = false
95 }
96
97 const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => {
98 const sequelizeOptions = { transaction: t }
99
100 // Save video object in database
101 const videoCreated = await video.save(sequelizeOptions)
102 videoCreated.VideoChannel = res.locals.videoChannel
103
104 // Set tags to the video
105 const tags = body.tags ? body.tags : youtubeDLInfo.tags
106 if (tags !== undefined) {
107 const tagInstances = await TagModel.findOrCreateTags(tags, t)
108
109 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
110 videoCreated.Tags = tagInstances
111 }
112
113 // Create video import object in database
114 const videoImport = await VideoImportModel.create({
115 targetUrl,
116 state: VideoImportState.PENDING,
117 videoId: videoCreated.id
118 }, sequelizeOptions)
119
120 videoImport.Video = videoCreated
121
122 return videoImport
123 })
124
125 // Create job to import the video
126 const payload = {
127 type: 'youtube-dl' as 'youtube-dl',
128 videoImportId: videoImport.id,
129 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
130 downloadThumbnail,
131 downloadPreview
132 }
133 await JobQueue.Instance.createJob({ type: 'video-import', payload })
134
135 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
136
137 return res.json(videoImport.toFormattedJSON()).end()
138}
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 101183eab..c9365da08 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -5,6 +5,7 @@ import { renamePromise } from '../../../helpers/core-utils'
5import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 5import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
6import { processImage } from '../../../helpers/image-utils' 6import { processImage } from '../../../helpers/image-utils'
7import { logger } from '../../../helpers/logger' 7import { logger } from '../../../helpers/logger'
8import { auditLoggerFactory, VideoAuditView } from '../../../helpers/audit-logger'
8import { getFormattedObjects, getServerActor, resetSequelizeInstance } from '../../../helpers/utils' 9import { getFormattedObjects, getServerActor, resetSequelizeInstance } from '../../../helpers/utils'
9import { 10import {
10 CONFIG, 11 CONFIG,
@@ -53,7 +54,9 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
53import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils' 54import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils'
54import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' 55import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
55import { videoCaptionsRouter } from './captions' 56import { videoCaptionsRouter } from './captions'
57import { videoImportsRouter } from './import'
56 58
59const auditLogger = auditLoggerFactory('videos')
57const videosRouter = express.Router() 60const videosRouter = express.Router()
58 61
59const reqVideoFileAdd = createReqFiles( 62const reqVideoFileAdd = createReqFiles(
@@ -79,6 +82,7 @@ videosRouter.use('/', blacklistRouter)
79videosRouter.use('/', rateVideoRouter) 82videosRouter.use('/', rateVideoRouter)
80videosRouter.use('/', videoCommentRouter) 83videosRouter.use('/', videoCommentRouter)
81videosRouter.use('/', videoCaptionsRouter) 84videosRouter.use('/', videoCaptionsRouter)
85videosRouter.use('/', videoImportsRouter)
82 86
83videosRouter.get('/categories', listVideoCategories) 87videosRouter.get('/categories', listVideoCategories)
84videosRouter.get('/licences', listVideoLicences) 88videosRouter.get('/licences', listVideoLicences)
@@ -158,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) {
158 const videoData = { 162 const videoData = {
159 name: videoInfo.name, 163 name: videoInfo.name,
160 remote: false, 164 remote: false,
161 extname: extname(videoPhysicalFile.filename),
162 category: videoInfo.category, 165 category: videoInfo.category,
163 licence: videoInfo.licence, 166 licence: videoInfo.licence,
164 language: videoInfo.language, 167 language: videoInfo.language,
@@ -247,6 +250,7 @@ async function addVideo (req: express.Request, res: express.Response) {
247 250
248 await federateVideoIfNeeded(video, true, t) 251 await federateVideoIfNeeded(video, true, t)
249 252
253 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
250 logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid) 254 logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid)
251 255
252 return videoCreated 256 return videoCreated
@@ -273,6 +277,7 @@ async function addVideo (req: express.Request, res: express.Response) {
273async function updateVideo (req: express.Request, res: express.Response) { 277async function updateVideo (req: express.Request, res: express.Response) {
274 const videoInstance: VideoModel = res.locals.video 278 const videoInstance: VideoModel = res.locals.video
275 const videoFieldsSave = videoInstance.toJSON() 279 const videoFieldsSave = videoInstance.toJSON()
280 const oldVideoAuditView = new VideoAuditView(videoInstance.toFormattedDetailsJSON())
276 const videoInfoToUpdate: VideoUpdate = req.body 281 const videoInfoToUpdate: VideoUpdate = req.body
277 const wasPrivateVideo = videoInstance.privacy === VideoPrivacy.PRIVATE 282 const wasPrivateVideo = videoInstance.privacy === VideoPrivacy.PRIVATE
278 283
@@ -344,9 +349,14 @@ async function updateVideo (req: express.Request, res: express.Response) {
344 349
345 const isNewVideo = wasPrivateVideo && videoInstanceUpdated.privacy !== VideoPrivacy.PRIVATE 350 const isNewVideo = wasPrivateVideo && videoInstanceUpdated.privacy !== VideoPrivacy.PRIVATE
346 await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t) 351 await federateVideoIfNeeded(videoInstanceUpdated, isNewVideo, t)
347 })
348 352
349 logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid) 353 auditLogger.update(
354 res.locals.oauth.token.User.Account.Actor.getIdentifier(),
355 new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()),
356 oldVideoAuditView
357 )
358 logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid)
359 })
350 } catch (err) { 360 } catch (err) {
351 // Force fields we want to update 361 // Force fields we want to update
352 // If the transaction is retried, sequelize will think the object has not changed 362 // If the transaction is retried, sequelize will think the object has not changed
@@ -423,6 +433,7 @@ async function removeVideo (req: express.Request, res: express.Response) {
423 await videoInstance.destroy({ transaction: t }) 433 await videoInstance.destroy({ transaction: t })
424 }) 434 })
425 435
436 auditLogger.delete(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoAuditView(videoInstance.toFormattedDetailsJSON()))
426 logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) 437 logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid)
427 438
428 return res.type('json').status(204).end() 439 return res.type('json').status(204).end()
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index d710f5c97..a9de11fb0 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -24,6 +24,7 @@ function activityPubContextify <T> (data: T) {
24 views: 'http://schema.org/Number', 24 views: 'http://schema.org/Number',
25 stats: 'http://schema.org/Number', 25 stats: 'http://schema.org/Number',
26 size: 'http://schema.org/Number', 26 size: 'http://schema.org/Number',
27 fps: 'http://schema.org/Number',
27 commentsEnabled: 'http://schema.org/Boolean', 28 commentsEnabled: 'http://schema.org/Boolean',
28 waitTranscoding: 'http://schema.org/Boolean', 29 waitTranscoding: 'http://schema.org/Boolean',
29 support: 'http://schema.org/Text' 30 support: 'http://schema.org/Text'
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
new file mode 100644
index 000000000..db20df20f
--- /dev/null
+++ b/server/helpers/audit-logger.ts
@@ -0,0 +1,265 @@
1import * as path from 'path'
2import { diff } from 'deep-object-diff'
3import { chain } from 'lodash'
4import * as flatten from 'flat'
5import * as winston from 'winston'
6import { CONFIG } from '../initializers'
7import { jsonLoggerFormat, labelFormatter } from './logger'
8import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared'
9import { VideoComment } from '../../shared/models/videos/video-comment.model'
10import { CustomConfig } from '../../shared/models/server/custom-config.model'
11
12enum AUDIT_TYPE {
13 CREATE = 'create',
14 UPDATE = 'update',
15 DELETE = 'delete'
16}
17
18const colors = winston.config.npm.colors
19colors.audit = winston.config.npm.colors.info
20
21winston.addColors(colors)
22
23const auditLogger = winston.createLogger({
24 levels: { audit: 0 },
25 transports: [
26 new winston.transports.File({
27 filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube-audit.log'),
28 level: 'audit',
29 maxsize: 5242880,
30 maxFiles: 5,
31 format: winston.format.combine(
32 winston.format.timestamp(),
33 labelFormatter,
34 winston.format.splat(),
35 jsonLoggerFormat
36 )
37 })
38 ],
39 exitOnError: true
40})
41
42function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) {
43 let entityInfos: object
44 if (action === AUDIT_TYPE.UPDATE && oldEntity) {
45 const oldEntityKeys = oldEntity.toLogKeys()
46 const diffObject = diff(oldEntityKeys, entity.toLogKeys())
47 const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => {
48 newKeys[`new-${entry[0]}`] = entry[1]
49 return newKeys
50 }, {})
51 entityInfos = { ...oldEntityKeys, ...diffKeys }
52 } else {
53 entityInfos = { ...entity.toLogKeys() }
54 }
55 auditLogger.log('audit', JSON.stringify({
56 user,
57 domain,
58 action,
59 ...entityInfos
60 }))
61}
62
63function auditLoggerFactory (domain: string) {
64 return {
65 create (user: string, entity: EntityAuditView) {
66 auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity)
67 },
68 update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) {
69 auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity)
70 },
71 delete (user: string, entity: EntityAuditView) {
72 auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity)
73 }
74 }
75}
76
77abstract class EntityAuditView {
78 constructor (private keysToKeep: Array<string>, private prefix: string, private entityInfos: object) { }
79 toLogKeys (): object {
80 return chain(flatten(this.entityInfos, { delimiter: '-', safe: true }))
81 .pick(this.keysToKeep)
82 .mapKeys((value, key) => `${this.prefix}-${key}`)
83 .value()
84 }
85}
86
87const videoKeysToKeep = [
88 'tags',
89 'uuid',
90 'id',
91 'uuid',
92 'createdAt',
93 'updatedAt',
94 'publishedAt',
95 'category',
96 'licence',
97 'language',
98 'privacy',
99 'description',
100 'duration',
101 'isLocal',
102 'name',
103 'thumbnailPath',
104 'previewPath',
105 'nsfw',
106 'waitTranscoding',
107 'account-id',
108 'account-uuid',
109 'account-name',
110 'channel-id',
111 'channel-uuid',
112 'channel-name',
113 'support',
114 'commentsEnabled'
115]
116class VideoAuditView extends EntityAuditView {
117 constructor (private video: VideoDetails) {
118 super(videoKeysToKeep, 'video', video)
119 }
120}
121
122const videoImportKeysToKeep = [
123 'id',
124 'targetUrl',
125 'video-name'
126]
127class VideoImportAuditView extends EntityAuditView {
128 constructor (private videoImport: VideoImport) {
129 super(videoImportKeysToKeep, 'video-import', videoImport)
130 }
131}
132
133const commentKeysToKeep = [
134 'id',
135 'text',
136 'threadId',
137 'inReplyToCommentId',
138 'videoId',
139 'createdAt',
140 'updatedAt',
141 'totalReplies',
142 'account-id',
143 'account-uuid',
144 'account-name'
145]
146class CommentAuditView extends EntityAuditView {
147 constructor (private comment: VideoComment) {
148 super(commentKeysToKeep, 'comment', comment)
149 }
150}
151
152const userKeysToKeep = [
153 'id',
154 'username',
155 'email',
156 'nsfwPolicy',
157 'autoPlayVideo',
158 'role',
159 'videoQuota',
160 'createdAt',
161 'account-id',
162 'account-uuid',
163 'account-name',
164 'account-followingCount',
165 'account-followersCount',
166 'account-createdAt',
167 'account-updatedAt',
168 'account-avatar-path',
169 'account-avatar-createdAt',
170 'account-avatar-updatedAt',
171 'account-displayName',
172 'account-description',
173 'videoChannels'
174]
175class UserAuditView extends EntityAuditView {
176 constructor (private user: User) {
177 super(userKeysToKeep, 'user', user)
178 }
179}
180
181const channelKeysToKeep = [
182 'id',
183 'uuid',
184 'name',
185 'followingCount',
186 'followersCount',
187 'createdAt',
188 'updatedAt',
189 'avatar-path',
190 'avatar-createdAt',
191 'avatar-updatedAt',
192 'displayName',
193 'description',
194 'support',
195 'isLocal',
196 'ownerAccount-id',
197 'ownerAccount-uuid',
198 'ownerAccount-name',
199 'ownerAccount-displayedName'
200]
201class VideoChannelAuditView extends EntityAuditView {
202 constructor (private channel: VideoChannel) {
203 super(channelKeysToKeep, 'channel', channel)
204 }
205}
206
207const videoAbuseKeysToKeep = [
208 'id',
209 'reason',
210 'reporterAccount',
211 'video-id',
212 'video-name',
213 'video-uuid',
214 'createdAt'
215]
216class VideoAbuseAuditView extends EntityAuditView {
217 constructor (private videoAbuse: VideoAbuse) {
218 super(videoAbuseKeysToKeep, 'abuse', videoAbuse)
219 }
220}
221
222const customConfigKeysToKeep = [
223 'instance-name',
224 'instance-shortDescription',
225 'instance-description',
226 'instance-terms',
227 'instance-defaultClientRoute',
228 'instance-defaultNSFWPolicy',
229 'instance-customizations-javascript',
230 'instance-customizations-css',
231 'services-twitter-username',
232 'services-twitter-whitelisted',
233 'cache-previews-size',
234 'cache-captions-size',
235 'signup-enabled',
236 'signup-limit',
237 'admin-email',
238 'user-videoQuota',
239 'transcoding-enabled',
240 'transcoding-threads',
241 'transcoding-resolutions'
242]
243class CustomConfigAuditView extends EntityAuditView {
244 constructor (customConfig: CustomConfig) {
245 const infos: any = customConfig
246 const resolutionsDict = infos.transcoding.resolutions
247 const resolutionsArray = []
248 Object.entries(resolutionsDict).forEach(([resolution, isEnabled]) => {
249 if (isEnabled) resolutionsArray.push(resolution)
250 })
251 Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
252 super(customConfigKeysToKeep, 'config', infos)
253 }
254}
255
256export {
257 auditLoggerFactory,
258 VideoImportAuditView,
259 VideoChannelAuditView,
260 CommentAuditView,
261 UserAuditView,
262 VideoAuditView,
263 VideoAbuseAuditView,
264 CustomConfigAuditView
265}
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 2951aef1e..884206aad 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -58,7 +58,7 @@ function escapeHTML (stringParam) {
58 '<': '&lt;', 58 '<': '&lt;',
59 '>': '&gt;', 59 '>': '&gt;',
60 '"': '&quot;', 60 '"': '&quot;',
61 "'": '&#39;', 61 '\'': '&#39;',
62 '/': '&#x2F;', 62 '/': '&#x2F;',
63 '`': '&#x60;', 63 '`': '&#x60;',
64 '=': '&#x3D;' 64 '=': '&#x3D;'
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index d97bbd2a9..b8075f3c7 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) {
45} 45}
46 46
47function sanitizeAndCheckVideoTorrentObject (video: any) { 47function sanitizeAndCheckVideoTorrentObject (video: any) {
48 if (video.type !== 'Video') return false 48 if (!video || video.type !== 'Video') return false
49 49
50 if (!setValidRemoteTags(video)) return false 50 if (!setValidRemoteTags(video)) return false
51 if (!setValidRemoteVideoUrls(video)) return false 51 if (!setValidRemoteVideoUrls(video)) return false
@@ -153,7 +153,8 @@ function isRemoteVideoUrlValid (url: any) {
153 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && 153 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
154 isActivityPubUrlValid(url.href) && 154 isActivityPubUrlValid(url.href) &&
155 validator.isInt(url.width + '', { min: 0 }) && 155 validator.isInt(url.width + '', { min: 0 }) &&
156 validator.isInt(url.size + '', { min: 0 }) 156 validator.isInt(url.size + '', { min: 0 }) &&
157 (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
157 ) || 158 ) ||
158 ( 159 (
159 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && 160 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
new file mode 100644
index 000000000..d8b9bfaff
--- /dev/null
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -0,0 +1,49 @@
1import 'express-validator'
2import 'multer'
3import * as validator from 'validator'
4import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
5import { exists } from './misc'
6import * as express from 'express'
7import { VideoChannelModel } from '../../models/video/video-channel'
8import { VideoImportModel } from '../../models/video/video-import'
9
10function isVideoImportTargetUrlValid (url: string) {
11 const isURLOptions = {
12 require_host: true,
13 require_tld: true,
14 require_protocol: true,
15 require_valid_protocol: true,
16 protocols: [ 'http', 'https' ]
17 }
18
19 return exists(url) &&
20 validator.isURL('' + url, isURLOptions) &&
21 validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
22}
23
24function isVideoImportStateValid (value: any) {
25 return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined
26}
27
28async function isVideoImportExist (id: number, res: express.Response) {
29 const videoImport = await VideoImportModel.loadAndPopulateVideo(id)
30
31 if (!videoImport) {
32 res.status(404)
33 .json({ error: 'Video import not found' })
34 .end()
35
36 return false
37 }
38
39 res.locals.videoImport = videoImport
40 return true
41}
42
43// ---------------------------------------------------------------------------
44
45export {
46 isVideoImportStateValid,
47 isVideoImportTargetUrlValid,
48 isVideoImportExist
49}
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index f0623c88b..ced56b82d 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,10 +1,11 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { join } from 'path' 2import { join } from 'path'
3import { VideoResolution } from '../../shared/models/videos' 3import { VideoResolution } from '../../shared/models/videos'
4import { CONFIG, VIDEO_TRANSCODING_FPS } from '../initializers' 4import { CONFIG, VIDEO_TRANSCODING_FPS, FFMPEG_NICE } from '../initializers'
5import { unlinkPromise } from './core-utils' 5import { unlinkPromise } from './core-utils'
6import { processImage } from './image-utils' 6import { processImage } from './image-utils'
7import { logger } from './logger' 7import { logger } from './logger'
8import { checkFFmpegEncoders } from '../initializers/checker'
8 9
9async function getVideoFileResolution (path: string) { 10async function getVideoFileResolution (path: string) {
10 const videoStream = await getVideoFileStream(path) 11 const videoStream = await getVideoFileStream(path)
@@ -55,7 +56,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
55 56
56 try { 57 try {
57 await new Promise<string>((res, rej) => { 58 await new Promise<string>((res, rej) => {
58 ffmpeg(fromPath) 59 ffmpeg(fromPath, { niceness: FFMPEG_NICE.THUMBNAIL })
59 .on('error', rej) 60 .on('error', rej)
60 .on('end', () => res(imageName)) 61 .on('end', () => res(imageName))
61 .thumbnail(options) 62 .thumbnail(options)
@@ -83,14 +84,14 @@ type TranscodeOptions = {
83 84
84function transcode (options: TranscodeOptions) { 85function transcode (options: TranscodeOptions) {
85 return new Promise<void>(async (res, rej) => { 86 return new Promise<void>(async (res, rej) => {
86 let command = ffmpeg(options.inputPath) 87 let command = ffmpeg(options.inputPath, { niceness: FFMPEG_NICE.TRANSCODING })
87 .output(options.outputPath) 88 .output(options.outputPath)
88 .videoCodec('libx264') 89 .preset(standard)
89 .outputOption('-threads ' + CONFIG.TRANSCODING.THREADS) 90
90 .outputOption('-movflags faststart') 91 if (CONFIG.TRANSCODING.THREADS > 0) {
91 .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorythm, 16 is optimal B-frames for it 92 // if we don't set any threads ffmpeg will chose automatically
92 .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16 93 command = command.outputOption('-threads ' + CONFIG.TRANSCODING.THREADS)
93 // .outputOption('-crf 18') 94 }
94 95
95 let fps = await getVideoFileFPS(options.inputPath) 96 let fps = await getVideoFileFPS(options.inputPath)
96 if (options.resolution !== undefined) { 97 if (options.resolution !== undefined) {
@@ -132,7 +133,8 @@ export {
132 getDurationFromVideoFile, 133 getDurationFromVideoFile,
133 generateImageFromVideoFile, 134 generateImageFromVideoFile,
134 transcode, 135 transcode,
135 getVideoFileFPS 136 getVideoFileFPS,
137 audio
136} 138}
137 139
138// --------------------------------------------------------------------------- 140// ---------------------------------------------------------------------------
@@ -149,3 +151,136 @@ function getVideoFileStream (path: string) {
149 }) 151 })
150 }) 152 })
151} 153}
154
155/**
156 * A slightly customised version of the 'veryfast' x264 preset
157 *
158 * The veryfast preset is right in the sweet spot of performance
159 * and quality. Superfast and ultrafast will give you better
160 * performance, but then quality is noticeably worse.
161 */
162function veryfast (_ffmpeg) {
163 _ffmpeg
164 .preset(standard)
165 .outputOption('-preset:v veryfast')
166 .outputOption(['--aq-mode=2', '--aq-strength=1.3'])
167 /*
168 MAIN reference: https://slhck.info/video/2017/03/01/rate-control.html
169 Our target situation is closer to a livestream than a stream,
170 since we want to reduce as much a possible the encoding burden,
171 altough not to the point of a livestream where there is a hard
172 constraint on the frames per second to be encoded.
173
174 why '--aq-mode=2 --aq-strength=1.3' instead of '-profile:v main'?
175 Make up for most of the loss of grain and macroblocking
176 with less computing power.
177 */
178}
179
180/**
181 * A preset optimised for a stillimage audio video
182 */
183function audio (_ffmpeg) {
184 _ffmpeg
185 .preset(veryfast)
186 .outputOption('-tune stillimage')
187}
188
189/**
190 * A toolbox to play with audio
191 */
192namespace audio {
193 export const get = (_ffmpeg, pos: number | string = 0) => {
194 // without position, ffprobe considers the last input only
195 // we make it consider the first input only
196 // if you pass a file path to pos, then ffprobe acts on that file directly
197 return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
198 _ffmpeg.ffprobe(pos, (err,data) => {
199 if (err) return rej(err)
200
201 if ('streams' in data) {
202 const audioStream = data['streams'].find(stream => stream['codec_type'] === 'audio')
203 if (audioStream) {
204 return res({
205 absolutePath: data.format.filename,
206 audioStream
207 })
208 }
209 }
210 return res({ absolutePath: data.format.filename })
211 })
212 })
213 }
214
215 export namespace bitrate {
216 export const baseKbitrate = 384
217
218 const toBits = (kbits: number): number => { return kbits * 8000 }
219
220 export const aac = (bitrate: number): number => {
221 switch (true) {
222 case bitrate > toBits(baseKbitrate):
223 return baseKbitrate
224 default:
225 return -1 // we interpret it as a signal to copy the audio stream as is
226 }
227 }
228
229 export const mp3 = (bitrate: number): number => {
230 /*
231 a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
232 That's why, when using aac, we can go to lower kbit/sec. The equivalences
233 made here are not made to be accurate, especially with good mp3 encoders.
234 */
235 switch (true) {
236 case bitrate <= toBits(192):
237 return 128
238 case bitrate <= toBits(384):
239 return 256
240 default:
241 return baseKbitrate
242 }
243 }
244 }
245}
246
247/**
248 * Standard profile, with variable bitrate audio and faststart.
249 *
250 * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
251 * See https://trac.ffmpeg.org/wiki/Encode/AAC#fdk_vbr
252 */
253async function standard (_ffmpeg) {
254 let _bitrate = audio.bitrate.baseKbitrate
255 let localFfmpeg = _ffmpeg
256 .format('mp4')
257 .videoCodec('libx264')
258 .outputOption('-level 3.1') // 3.1 is the minimal ressource allocation for our highest supported resolution
259 .outputOption('-b_strategy 1') // NOTE: b-strategy 1 - heuristic algorythm, 16 is optimal B-frames for it
260 .outputOption('-bf 16') // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16
261 .outputOption('-map_metadata -1') // strip all metadata
262 .outputOption('-movflags faststart')
263 const _audio = await audio.get(localFfmpeg)
264
265 if (!_audio.audioStream) {
266 return localFfmpeg.noAudio()
267 }
268
269 // we try to reduce the ceiling bitrate by making rough correspondances of bitrates
270 // of course this is far from perfect, but it might save some space in the end
271 if (audio.bitrate[_audio.audioStream['codec_name']]) {
272 _bitrate = audio.bitrate[_audio.audioStream['codec_name']](_audio.audioStream['bit_rate'])
273 if (_bitrate === -1) {
274 return localFfmpeg.audioCodec('copy')
275 }
276 }
277
278 // we favor VBR, if a good AAC encoder is available
279 if ((await checkFFmpegEncoders()).get('libfdk_aac')) {
280 return localFfmpeg
281 .audioCodec('libfdk_aac')
282 .audioQuality(5)
283 }
284
285 return localFfmpeg.audioBitrate(_bitrate)
286}
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 6d369a8fb..480c5b49e 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) {
22} 22}
23 23
24const consoleLoggerFormat = winston.format.printf(info => { 24const consoleLoggerFormat = winston.format.printf(info => {
25 let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2) 25 let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2)
26 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' 26 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
27 else additionalInfos = ' ' + additionalInfos 27 else additionalInfos = ' ' + additionalInfos
28 28
@@ -96,13 +96,13 @@ const bunyanLogger = {
96 error: bunyanLogFactory('error'), 96 error: bunyanLogFactory('error'),
97 fatal: bunyanLogFactory('error') 97 fatal: bunyanLogFactory('error')
98} 98}
99
100// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
101 100
102export { 101export {
103 timestampFormatter, 102 timestampFormatter,
104 labelFormatter, 103 labelFormatter,
105 consoleLoggerFormat, 104 consoleLoggerFormat,
105 jsonLoggerFormat,
106 logger, 106 logger,
107 bunyanLogger 107 bunyanLogger
108} 108}
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index cfb427570..7abcec5d7 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -6,11 +6,35 @@ import { CONFIG } from '../initializers'
6import { UserModel } from '../models/account/user' 6import { UserModel } from '../models/account/user'
7import { ActorModel } from '../models/activitypub/actor' 7import { ActorModel } from '../models/activitypub/actor'
8import { ApplicationModel } from '../models/application/application' 8import { ApplicationModel } from '../models/application/application'
9import { pseudoRandomBytesPromise } from './core-utils' 9import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils'
10import { logger } from './logger' 10import { logger } from './logger'
11import { isArray } from './custom-validators/misc'
11 12
12const isCidr = require('is-cidr') 13const isCidr = require('is-cidr')
13 14
15function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[] }) {
16 const files = req.files
17
18 if (!files) return
19
20 if (isArray(files)) {
21 (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
22 return
23 }
24
25 for (const key of Object.keys(files)) {
26 const file = files[key]
27
28 if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
29 else deleteFileAsync(file.path)
30 }
31}
32
33function deleteFileAsync (path: string) {
34 unlinkPromise(path)
35 .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
36}
37
14async function generateRandomString (size: number) { 38async function generateRandomString (size: number) {
15 const raw = await pseudoRandomBytesPromise(size) 39 const raw = await pseudoRandomBytesPromise(size)
16 40
@@ -162,6 +186,8 @@ type SortType = { sortModel: any, sortValue: string }
162// --------------------------------------------------------------------------- 186// ---------------------------------------------------------------------------
163 187
164export { 188export {
189 cleanUpReqFiles,
190 deleteFileAsync,
165 generateRandomString, 191 generateRandomString,
166 getFormattedObjects, 192 getFormattedObjects,
167 isSignupAllowed, 193 isSignupAllowed,
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
new file mode 100644
index 000000000..c59ab9de0
--- /dev/null
+++ b/server/helpers/youtube-dl.ts
@@ -0,0 +1,142 @@
1import * as youtubeDL from 'youtube-dl'
2import { truncate } from 'lodash'
3import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
4import { join } from 'path'
5import * as crypto from 'crypto'
6import { logger } from './logger'
7
8export type YoutubeDLInfo = {
9 name: string
10 description: string
11 category: number
12 licence: number
13 nsfw: boolean
14 tags: string[]
15 thumbnailUrl: string
16}
17
18function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
19 return new Promise<YoutubeDLInfo>((res, rej) => {
20 const options = [ '-j', '--flat-playlist' ]
21
22 youtubeDL.getInfo(url, options, (err, info) => {
23 if (err) return rej(err)
24
25 const obj = normalizeObject(info)
26
27 return res(buildVideoInfo(obj))
28 })
29 })
30}
31
32function downloadYoutubeDLVideo (url: string) {
33 const hash = crypto.createHash('sha256').update(url).digest('hex')
34 const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
35
36 logger.info('Importing video %s', url)
37
38 const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
39
40 return new Promise<string>((res, rej) => {
41 youtubeDL.exec(url, options, async (err, output) => {
42 if (err) return rej(err)
43
44 return res(path)
45 })
46 })
47}
48
49// ---------------------------------------------------------------------------
50
51export {
52 downloadYoutubeDLVideo,
53 getYoutubeDLInfo
54}
55
56// ---------------------------------------------------------------------------
57
58function normalizeObject (obj: any) {
59 const newObj: any = {}
60
61 for (const key of Object.keys(obj)) {
62 // Deprecated key
63 if (key === 'resolution') continue
64
65 const value = obj[key]
66
67 if (typeof value === 'string') {
68 newObj[key] = value.normalize()
69 } else {
70 newObj[key] = value
71 }
72 }
73
74 return newObj
75}
76
77function buildVideoInfo (obj: any) {
78 return {
79 name: titleTruncation(obj.title),
80 description: descriptionTruncation(obj.description),
81 category: getCategory(obj.categories),
82 licence: getLicence(obj.license),
83 nsfw: isNSFW(obj),
84 tags: getTags(obj.tags),
85 thumbnailUrl: obj.thumbnail || undefined
86 }
87}
88
89function titleTruncation (title: string) {
90 return truncate(title, {
91 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
92 'separator': /,? +/,
93 'omission': ' […]'
94 })
95}
96
97function descriptionTruncation (description: string) {
98 if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
99
100 return truncate(description, {
101 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
102 'separator': /,? +/,
103 'omission': ' […]'
104 })
105}
106
107function isNSFW (info: any) {
108 return info.age_limit && info.age_limit >= 16
109}
110
111function getTags (tags: any) {
112 if (Array.isArray(tags) === false) return []
113
114 return tags
115 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
116 .map(t => t.normalize())
117 .slice(0, 5)
118}
119
120function getLicence (licence: string) {
121 if (!licence) return undefined
122
123 if (licence.indexOf('Creative Commons Attribution') !== -1) return 1
124
125 return undefined
126}
127
128function getCategory (categories: string[]) {
129 if (!categories) return undefined
130
131 const categoryString = categories[0]
132 if (!categoryString || typeof categoryString !== 'string') return undefined
133
134 if (categoryString === 'News & Politics') return 11
135
136 for (const key of Object.keys(VIDEO_CATEGORIES)) {
137 const category = VIDEO_CATEGORIES[key]
138 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
139 }
140
141 return undefined
142}
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 52a1aeb50..608123607 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -43,7 +43,7 @@ function checkMissedConfig () {
43 const required = [ 'listen.port', 'listen.hostname', 43 const required = [ 'listen.port', 'listen.hostname',
44 'webserver.https', 'webserver.hostname', 'webserver.port', 44 'webserver.https', 'webserver.hostname', 'webserver.port',
45 'trust_proxy', 45 'trust_proxy',
46 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 46 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
47 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 47 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
48 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 48 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
49 'log.level', 49 'log.level',
@@ -51,6 +51,7 @@ function checkMissedConfig () {
51 'cache.previews.size', 'admin.email', 51 'cache.previews.size', 'admin.email',
52 'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 52 'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
53 'transcoding.enabled', 'transcoding.threads', 53 'transcoding.enabled', 'transcoding.threads',
54 'import.videos.http.enabled',
54 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', 55 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
55 'instance.default_nsfw_policy', 'instance.robots', 56 'instance.default_nsfw_policy', 'instance.robots',
56 'services.twitter.username', 'services.twitter.whitelisted' 57 'services.twitter.username', 'services.twitter.whitelisted'
@@ -84,11 +85,11 @@ function checkMissedConfig () {
84async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { 85async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
85 const Ffmpeg = require('fluent-ffmpeg') 86 const Ffmpeg = require('fluent-ffmpeg')
86 const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) 87 const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs)
87
88 const codecs = await getAvailableCodecsPromise() 88 const codecs = await getAvailableCodecsPromise()
89 const canEncode = [ 'libx264' ]
90
89 if (CONFIG.TRANSCODING.ENABLED === false) return undefined 91 if (CONFIG.TRANSCODING.ENABLED === false) return undefined
90 92
91 const canEncode = [ 'libx264' ]
92 for (const codec of canEncode) { 93 for (const codec of canEncode) {
93 if (codecs[codec] === undefined) { 94 if (codecs[codec] === undefined) {
94 throw new Error('Unknown codec ' + codec + ' in FFmpeg.') 95 throw new Error('Unknown codec ' + codec + ' in FFmpeg.')
@@ -98,6 +99,29 @@ async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
98 throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg') 99 throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg')
99 } 100 }
100 } 101 }
102
103 checkFFmpegEncoders()
104}
105
106// Optional encoders, if present, can be used to improve transcoding
107// Here we ask ffmpeg if it detects their presence on the system, so that we can later use them
108let supportedOptionalEncoders: Map<string, boolean>
109async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
110 if (supportedOptionalEncoders !== undefined) {
111 return supportedOptionalEncoders
112 }
113
114 const Ffmpeg = require('fluent-ffmpeg')
115 const getAvailableEncodersPromise = promisify0(Ffmpeg.getAvailableEncoders)
116 const encoders = await getAvailableEncodersPromise()
117 const optionalEncoders = [ 'libfdk_aac' ]
118 supportedOptionalEncoders = new Map<string, boolean>()
119
120 for (const encoder of optionalEncoders) {
121 supportedOptionalEncoders.set(encoder,
122 encoders[encoder] !== undefined
123 )
124 }
101} 125}
102 126
103// We get db by param to not import it in this file (import orders) 127// We get db by param to not import it in this file (import orders)
@@ -126,6 +150,7 @@ async function applicationExist () {
126export { 150export {
127 checkConfig, 151 checkConfig,
128 checkFFmpeg, 152 checkFFmpeg,
153 checkFFmpegEncoders,
129 checkMissedConfig, 154 checkMissedConfig,
130 clientsExist, 155 clientsExist,
131 usersExist, 156 usersExist,
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 3aa979668..069d9b2e8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos'
8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' 8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
10import { invert } from 'lodash' 10import { invert } from 'lodash'
11import { VideoImportState } from '../../shared/models/videos/video-import-state.enum'
11 12
12// Use a variable to reload the configuration if we need 13// Use a variable to reload the configuration if we need
13let config: IConfig = require('config') 14let config: IConfig = require('config')
@@ -36,6 +37,7 @@ const SORTABLE_COLUMNS = {
36 VIDEO_ABUSES: [ 'id', 'createdAt' ], 37 VIDEO_ABUSES: [ 'id', 'createdAt' ],
37 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], 38 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
38 VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ], 39 VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ],
40 VIDEO_IMPORTS: [ 'createdAt' ],
39 VIDEO_COMMENT_THREADS: [ 'createdAt' ], 41 VIDEO_COMMENT_THREADS: [ 'createdAt' ],
40 BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ], 42 BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ],
41 FOLLOWERS: [ 'createdAt' ], 43 FOLLOWERS: [ 'createdAt' ],
@@ -85,6 +87,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
85 'activitypub-follow': 5, 87 'activitypub-follow': 5,
86 'video-file-import': 1, 88 'video-file-import': 1,
87 'video-file': 1, 89 'video-file': 1,
90 'video-import': 1,
88 'email': 5 91 'email': 5
89} 92}
90const JOB_CONCURRENCY: { [ id in JobType ]: number } = { 93const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
@@ -94,18 +97,29 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
94 'activitypub-follow': 3, 97 'activitypub-follow': 3,
95 'video-file-import': 1, 98 'video-file-import': 1,
96 'video-file': 1, 99 'video-file': 1,
100 'video-import': 1,
97 'email': 5 101 'email': 5
98} 102}
103const JOB_TTL: { [ id in JobType ]: number } = {
104 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
105 'activitypub-http-unicast': 60000 * 10, // 10 minutes
106 'activitypub-http-fetcher': 60000 * 10, // 10 minutes
107 'activitypub-follow': 60000 * 10, // 10 minutes
108 'video-file-import': 1000 * 3600, // 1 hour
109 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
110 'video-import': 1000 * 3600 * 5, // 5 hours
111 'email': 60000 * 10 // 10 minutes
112}
99const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job 113const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
100const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds 114const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
101const JOB_REQUEST_TTL = 60000 * 10 // 10 minutes
102const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 115const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
103 116
104// 1 hour 117// 1 hour
105let SCHEDULER_INTERVALS_MS = { 118let SCHEDULER_INTERVALS_MS = {
106 badActorFollow: 60000 * 60, // 1 hour 119 badActorFollow: 60000 * 60, // 1 hour
107 removeOldJobs: 60000 * 60, // 1 hour 120 removeOldJobs: 60000 * 60, // 1 hour
108 updateVideos: 60000 // 1 minute 121 updateVideos: 60000, // 1 minute
122 youtubeDLUpdate: 60000 * 60 * 24 // 1 day
109} 123}
110 124
111// --------------------------------------------------------------------------- 125// ---------------------------------------------------------------------------
@@ -121,7 +135,10 @@ const CONFIG = {
121 HOSTNAME: config.get<string>('database.hostname'), 135 HOSTNAME: config.get<string>('database.hostname'),
122 PORT: config.get<number>('database.port'), 136 PORT: config.get<number>('database.port'),
123 USERNAME: config.get<string>('database.username'), 137 USERNAME: config.get<string>('database.username'),
124 PASSWORD: config.get<string>('database.password') 138 PASSWORD: config.get<string>('database.password'),
139 POOL: {
140 MAX: config.get<number>('database.pool.max')
141 }
125 }, 142 },
126 REDIS: { 143 REDIS: {
127 HOSTNAME: config.has('redis.hostname') ? config.get<string>('redis.hostname') : null, 144 HOSTNAME: config.has('redis.hostname') ? config.get<string>('redis.hostname') : null,
@@ -189,6 +206,13 @@ const CONFIG = {
189 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } 206 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
190 } 207 }
191 }, 208 },
209 IMPORT: {
210 VIDEOS: {
211 HTTP: {
212 get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }
213 }
214 }
215 },
192 CACHE: { 216 CACHE: {
193 PREVIEWS: { 217 PREVIEWS: {
194 get SIZE () { return config.get<number>('cache.previews.size') } 218 get SIZE () { return config.get<number>('cache.previews.size') }
@@ -245,6 +269,9 @@ const CONSTRAINTS_FIELDS = {
245 } 269 }
246 } 270 }
247 }, 271 },
272 VIDEO_IMPORTS: {
273 URL: { min: 3, max: 2000 } // Length
274 },
248 VIDEOS: { 275 VIDEOS: {
249 NAME: { min: 3, max: 120 }, // Length 276 NAME: { min: 3, max: 120 }, // Length
250 LANGUAGE: { min: 1, max: 10 }, // Length 277 LANGUAGE: { min: 1, max: 10 }, // Length
@@ -259,7 +286,7 @@ const CONSTRAINTS_FIELDS = {
259 }, 286 },
260 EXTNAME: [ '.mp4', '.ogv', '.webm' ], 287 EXTNAME: [ '.mp4', '.ogv', '.webm' ],
261 INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 288 INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
262 DURATION: { min: 1 }, // Number 289 DURATION: { min: 0 }, // Number
263 TAGS: { min: 0, max: 5 }, // Number of total tags 290 TAGS: { min: 0, max: 5 }, // Number of total tags
264 TAG: { min: 2, max: 30 }, // Length 291 TAG: { min: 2, max: 30 }, // Length
265 THUMBNAIL: { min: 2, max: 30 }, 292 THUMBNAIL: { min: 2, max: 30 },
@@ -313,6 +340,11 @@ const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = {
313 DISLIKE: 'dislike' 340 DISLIKE: 'dislike'
314} 341}
315 342
343const FFMPEG_NICE: { [ id: string ]: number } = {
344 THUMBNAIL: 2, // 2 just for don't blocking servers
345 TRANSCODING: 15
346}
347
316const VIDEO_CATEGORIES = { 348const VIDEO_CATEGORIES = {
317 1: 'Music', 349 1: 'Music',
318 2: 'Films', 350 2: 'Films',
@@ -355,7 +387,14 @@ const VIDEO_PRIVACIES = {
355 387
356const VIDEO_STATES = { 388const VIDEO_STATES = {
357 [VideoState.PUBLISHED]: 'Published', 389 [VideoState.PUBLISHED]: 'Published',
358 [VideoState.TO_TRANSCODE]: 'To transcode' 390 [VideoState.TO_TRANSCODE]: 'To transcode',
391 [VideoState.TO_IMPORT]: 'To import'
392}
393
394const VIDEO_IMPORT_STATES = {
395 [VideoImportState.FAILED]: 'Failed',
396 [VideoImportState.PENDING]: 'Pending',
397 [VideoImportState.SUCCESS]: 'Success'
359} 398}
360 399
361const VIDEO_MIMETYPE_EXT = { 400const VIDEO_MIMETYPE_EXT = {
@@ -553,6 +592,7 @@ export {
553 ROUTE_CACHE_LIFETIME, 592 ROUTE_CACHE_LIFETIME,
554 SORTABLE_COLUMNS, 593 SORTABLE_COLUMNS,
555 FEEDS, 594 FEEDS,
595 JOB_TTL,
556 NSFW_POLICY_TYPES, 596 NSFW_POLICY_TYPES,
557 STATIC_MAX_AGE, 597 STATIC_MAX_AGE,
558 STATIC_PATHS, 598 STATIC_PATHS,
@@ -567,8 +607,8 @@ export {
567 VIDEO_RATE_TYPES, 607 VIDEO_RATE_TYPES,
568 VIDEO_MIMETYPE_EXT, 608 VIDEO_MIMETYPE_EXT,
569 VIDEO_TRANSCODING_FPS, 609 VIDEO_TRANSCODING_FPS,
610 FFMPEG_NICE,
570 JOB_REQUEST_TIMEOUT, 611 JOB_REQUEST_TIMEOUT,
571 JOB_REQUEST_TTL,
572 USER_PASSWORD_RESET_LIFETIME, 612 USER_PASSWORD_RESET_LIFETIME,
573 IMAGE_MIMETYPE_EXT, 613 IMAGE_MIMETYPE_EXT,
574 SCHEDULER_INTERVALS_MS, 614 SCHEDULER_INTERVALS_MS,
@@ -576,6 +616,7 @@ export {
576 RATES_LIMIT, 616 RATES_LIMIT,
577 VIDEO_EXT_MIMETYPE, 617 VIDEO_EXT_MIMETYPE,
578 JOB_COMPLETED_LIFETIME, 618 JOB_COMPLETED_LIFETIME,
619 VIDEO_IMPORT_STATES,
579 VIDEO_VIEW_LIFETIME, 620 VIDEO_VIEW_LIFETIME,
580 buildLanguages 621 buildLanguages
581} 622}
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 1a9ce5a61..0be752363 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag'
24import { CONFIG } from './constants' 24import { CONFIG } from './constants'
25import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' 25import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update'
26import { VideoCaptionModel } from '../models/video/video-caption' 26import { VideoCaptionModel } from '../models/video/video-caption'
27import { VideoImportModel } from '../models/video/video-import'
27 28
28require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 29require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
29 30
@@ -32,6 +33,7 @@ const username = CONFIG.DATABASE.USERNAME
32const password = CONFIG.DATABASE.PASSWORD 33const password = CONFIG.DATABASE.PASSWORD
33const host = CONFIG.DATABASE.HOSTNAME 34const host = CONFIG.DATABASE.HOSTNAME
34const port = CONFIG.DATABASE.PORT 35const port = CONFIG.DATABASE.PORT
36const poolMax = CONFIG.DATABASE.POOL.MAX
35 37
36const sequelizeTypescript = new SequelizeTypescript({ 38const sequelizeTypescript = new SequelizeTypescript({
37 database: dbname, 39 database: dbname,
@@ -40,6 +42,9 @@ const sequelizeTypescript = new SequelizeTypescript({
40 port, 42 port,
41 username, 43 username,
42 password, 44 password,
45 pool: {
46 max: poolMax
47 },
43 benchmark: isTestInstance(), 48 benchmark: isTestInstance(),
44 isolationLevel: SequelizeTypescript.Transaction.ISOLATION_LEVELS.SERIALIZABLE, 49 isolationLevel: SequelizeTypescript.Transaction.ISOLATION_LEVELS.SERIALIZABLE,
45 operatorsAliases: false, 50 operatorsAliases: false,
@@ -77,7 +82,8 @@ async function initDatabaseModels (silent: boolean) {
77 VideoTagModel, 82 VideoTagModel,
78 VideoModel, 83 VideoModel,
79 VideoCommentModel, 84 VideoCommentModel,
80 ScheduleVideoUpdateModel 85 ScheduleVideoUpdateModel,
86 VideoImportModel
81 ]) 87 ])
82 88
83 // Check extensions exist in the database 89 // Check extensions exist in the database
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts
index 62791ff1b..82b661a03 100644
--- a/server/lib/activitypub/process/process-update.ts
+++ b/server/lib/activitypub/process/process-update.ts
@@ -108,7 +108,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate)
108 await Promise.all(videoFileDestroyTasks) 108 await Promise.all(videoFileDestroyTasks)
109 109
110 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject) 110 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject)
111 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f)) 111 const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions))
112 await Promise.all(tasks) 112 await Promise.all(tasks)
113 113
114 // Update Tags 114 // Update Tags
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index b3fbf88d0..e2f46bd02 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -162,7 +162,8 @@ function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObje
162 infoHash: parsed.infoHash, 162 infoHash: parsed.infoHash,
163 resolution: fileUrl.width, 163 resolution: fileUrl.width,
164 size: fileUrl.size, 164 size: fileUrl.size,
165 videoId: videoCreated.id 165 videoId: videoCreated.id,
166 fps: fileUrl.fps
166 } as VideoFileModel 167 } as VideoFileModel
167 attributes.push(attribute) 168 attributes.push(attribute)
168 } 169 }
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
new file mode 100644
index 000000000..cdfe412cc
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -0,0 +1,145 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
4import { VideoImportModel } from '../../../models/video/video-import'
5import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file'
9import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils'
10import { CONFIG, sequelizeTypescript } from '../../../initializers'
11import { doRequestAndSaveToFile } from '../../../helpers/requests'
12import { VideoState } from '../../../../shared'
13import { JobQueue } from '../index'
14import { federateVideoIfNeeded } from '../../activitypub'
15import { VideoModel } from '../../../models/video/video'
16
17export type VideoImportPayload = {
18 type: 'youtube-dl'
19 videoImportId: number
20 thumbnailUrl: string
21 downloadThumbnail: boolean
22 downloadPreview: boolean
23}
24
25async function processVideoImport (job: Bull.Job) {
26 const payload = job.data as VideoImportPayload
27 logger.info('Processing video import in job %d.', job.id)
28
29 const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
30 if (!videoImport || !videoImport.Video) {
31 throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
32 }
33
34 let tempVideoPath: string
35 let videoDestFile: string
36 let videoFile: VideoFileModel
37 try {
38 // Download video from youtubeDL
39 tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
40
41 // Get information about this video
42 const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
43 const fps = await getVideoFileFPS(tempVideoPath)
44 const stats = await statPromise(tempVideoPath)
45 const duration = await getDurationFromVideoFile(tempVideoPath)
46
47 // Create video file object in database
48 const videoFileData = {
49 extname: extname(tempVideoPath),
50 resolution: videoFileResolution,
51 size: stats.size,
52 fps,
53 videoId: videoImport.videoId
54 }
55 videoFile = new VideoFileModel(videoFileData)
56 // Import if the import fails, to clean files
57 videoImport.Video.VideoFiles = [ videoFile ]
58
59 // Move file
60 videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
61 await renamePromise(tempVideoPath, videoDestFile)
62 tempVideoPath = null // This path is not used anymore
63
64 // Process thumbnail
65 if (payload.downloadThumbnail) {
66 if (payload.thumbnailUrl) {
67 const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
68 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
69 } else {
70 await videoImport.Video.createThumbnail(videoFile)
71 }
72 }
73
74 // Process preview
75 if (payload.downloadPreview) {
76 if (payload.thumbnailUrl) {
77 const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
78 await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
79 } else {
80 await videoImport.Video.createPreview(videoFile)
81 }
82 }
83
84 // Create torrent
85 await videoImport.Video.createTorrentAndSetInfoHash(videoFile)
86
87 const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => {
88 // Refresh video
89 const video = await VideoModel.load(videoImport.videoId, t)
90 if (!video) throw new Error('Video linked to import ' + videoImport.videoId + ' does not exist anymore.')
91 videoImport.Video = video
92
93 const videoFileCreated = await videoFile.save({ transaction: t })
94 video.VideoFiles = [ videoFileCreated ]
95
96 // Update video DB object
97 video.duration = duration
98 video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
99 const videoUpdated = await video.save({ transaction: t })
100
101 // Now we can federate the video (reload from database, we need more attributes)
102 const videoForFederation = await VideoModel.loadByUUIDAndPopulateAccountAndServerAndTags(video.uuid, t)
103 await federateVideoIfNeeded(videoForFederation, true, t)
104
105 // Update video import object
106 videoImport.state = VideoImportState.SUCCESS
107 const videoImportUpdated = await videoImport.save({ transaction: t })
108
109 logger.info('Video %s imported.', videoImport.targetUrl)
110
111 videoImportUpdated.Video = videoUpdated
112 return videoImportUpdated
113 })
114
115 // Create transcoding jobs?
116 if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
117 // Put uuid because we don't have id auto incremented for now
118 const dataInput = {
119 videoUUID: videoImportUpdated.Video.uuid,
120 isNewVideo: true
121 }
122
123 await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
124 }
125
126 } catch (err) {
127 try {
128 if (tempVideoPath) await unlinkPromise(tempVideoPath)
129 } catch (errUnlink) {
130 logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
131 }
132
133 videoImport.error = err.message
134 videoImport.state = VideoImportState.FAILED
135 await videoImport.save()
136
137 throw err
138 }
139}
140
141// ---------------------------------------------------------------------------
142
143export {
144 processVideoImport
145}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8ff0c169e..8a24604e1 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -2,13 +2,14 @@ import * as Bull from 'bull'
2import { JobState, JobType } from '../../../shared/models' 2import { JobState, JobType } from '../../../shared/models'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { Redis } from '../redis' 4import { Redis } from '../redis'
5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_REQUEST_TTL } from '../../initializers' 5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers'
6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' 6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
12 13
13type CreateJobArgument = 14type CreateJobArgument =
14 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 15 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -17,7 +18,8 @@ type CreateJobArgument =
17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 18 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
18 { type: 'video-file-import', payload: VideoFileImportPayload } | 19 { type: 'video-file-import', payload: VideoFileImportPayload } |
19 { type: 'video-file', payload: VideoFilePayload } | 20 { type: 'video-file', payload: VideoFilePayload } |
20 { type: 'email', payload: EmailPayload } 21 { type: 'email', payload: EmailPayload } |
22 { type: 'video-import', payload: VideoImportPayload }
21 23
22const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { 24const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
23 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 25 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
@@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
26 'activitypub-follow': processActivityPubFollow, 28 'activitypub-follow': processActivityPubFollow,
27 'video-file-import': processVideoFileImport, 29 'video-file-import': processVideoFileImport,
28 'video-file': processVideoFile, 30 'video-file': processVideoFile,
29 'email': processEmail 31 'email': processEmail,
32 'video-import': processVideoImport
30} 33}
31 34
32const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = { 35const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
@@ -43,7 +46,8 @@ const jobTypes: JobType[] = [
43 'activitypub-http-unicast', 46 'activitypub-http-unicast',
44 'email', 47 'email',
45 'video-file', 48 'video-file',
46 'video-file-import' 49 'video-file-import',
50 'video-import'
47] 51]
48 52
49class JobQueue { 53class JobQueue {
@@ -75,7 +79,11 @@ class JobQueue {
75 const handler = handlers[handlerName] 79 const handler = handlers[handlerName]
76 80
77 queue.process(JOB_CONCURRENCY[handlerName], handler) 81 queue.process(JOB_CONCURRENCY[handlerName], handler)
78 .catch(err => logger.error('Cannot execute job queue %s.', handlerName, { err })) 82 .catch(err => logger.error('Error in job queue processor %s.', handlerName, { err }))
83
84 queue.on('failed', (job, err) => {
85 logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err })
86 })
79 87
80 queue.on('error', err => { 88 queue.on('error', err => {
81 logger.error('Error in job queue %s.', handlerName, { err }) 89 logger.error('Error in job queue %s.', handlerName, { err })
@@ -102,11 +110,8 @@ class JobQueue {
102 110
103 const jobArgs: Bull.JobOptions = { 111 const jobArgs: Bull.JobOptions = {
104 backoff: { delay: 60 * 1000, type: 'exponential' }, 112 backoff: { delay: 60 * 1000, type: 'exponential' },
105 attempts: JOB_ATTEMPTS[obj.type] 113 attempts: JOB_ATTEMPTS[obj.type],
106 } 114 timeout: JOB_TTL[obj.type]
107
108 if (jobsWithRequestTimeout[obj.type] === true) {
109 jobArgs.timeout = JOB_REQUEST_TTL
110 } 115 }
111 116
112 return queue.add(obj.payload, jobArgs) 117 return queue.add(obj.payload, jobArgs)
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
new file mode 100644
index 000000000..a2d919603
--- /dev/null
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -0,0 +1,72 @@
1// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
2// We rewrote it to avoid sync calls
3
4import { AbstractScheduler } from './abstract-scheduler'
5import { SCHEDULER_INTERVALS_MS } from '../../initializers'
6import { logger } from '../../helpers/logger'
7import * as request from 'request'
8import { createWriteStream, writeFile } from 'fs'
9import { join } from 'path'
10import { root } from '../../helpers/core-utils'
11
12export class YoutubeDlUpdateScheduler extends AbstractScheduler {
13
14 private static instance: AbstractScheduler
15
16 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.youtubeDLUpdate
17
18 private constructor () {
19 super()
20 }
21
22 async execute () {
23 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
24 const bin = join(binDirectory, 'youtube-dl')
25 const detailsPath = join(binDirectory, 'details')
26 const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
27
28 request.get(url, { followRedirect: false }, (err, res) => {
29 if (err) {
30 logger.error('Cannot update youtube-dl.', { err })
31 return
32 }
33
34 if (res.statusCode !== 302) {
35 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', res.statusCode)
36 return
37 }
38
39 const url = res.headers.location
40 const downloadFile = request.get(url)
41 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
42
43 downloadFile.on('response', res => {
44 if (res.statusCode !== 200) {
45 logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', res.statusCode)
46 return
47 }
48
49 downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
50 })
51
52 downloadFile.on('error', err => logger.error('youtube-dl update error.', { err }))
53
54 downloadFile.on('end', () => {
55 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
56 writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
57 if (err) {
58 logger.error('youtube-dl update error: cannot write details.', { err })
59 return
60 }
61
62 logger.info('youtube-dl updated to version %s.', newVersion)
63 })
64 })
65
66 })
67 }
68
69 static get Instance () {
70 return this.instance || (this.instance = new this())
71 }
72}
diff --git a/server/lib/user.ts b/server/lib/user.ts
index ac5f55260..e7a45f5aa 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -17,6 +17,7 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse
17 17
18 const userCreated = await userToCreate.save(userOptions) 18 const userCreated = await userToCreate.save(userOptions)
19 const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t) 19 const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t)
20 userCreated.Account = accountCreated
20 21
21 const videoChannelDisplayName = `Default ${userCreated.username} channel` 22 const videoChannelDisplayName = `Default ${userCreated.username} channel`
22 const videoChannelInfo = { 23 const videoChannelInfo = {
diff --git a/server/middlewares/validators/avatar.ts b/server/middlewares/validators/avatar.ts
index f346ea92f..5860735c6 100644
--- a/server/middlewares/validators/avatar.ts
+++ b/server/middlewares/validators/avatar.ts
@@ -4,6 +4,7 @@ import { isAvatarFile } from '../../helpers/custom-validators/users'
4import { areValidationErrors } from './utils' 4import { areValidationErrors } from './utils'
5import { CONSTRAINTS_FIELDS } from '../../initializers' 5import { CONSTRAINTS_FIELDS } from '../../initializers'
6import { logger } from '../../helpers/logger' 6import { logger } from '../../helpers/logger'
7import { cleanUpReqFiles } from '../../helpers/utils'
7 8
8const updateAvatarValidator = [ 9const updateAvatarValidator = [
9 body('avatarfile').custom((value, { req }) => isAvatarFile(req.files)).withMessage( 10 body('avatarfile').custom((value, { req }) => isAvatarFile(req.files)).withMessage(
@@ -14,7 +15,7 @@ const updateAvatarValidator = [
14 (req: express.Request, res: express.Response, next: express.NextFunction) => { 15 (req: express.Request, res: express.Response, next: express.NextFunction) => {
15 logger.debug('Checking updateAvatarValidator parameters', { files: req.files }) 16 logger.debug('Checking updateAvatarValidator parameters', { files: req.files })
16 17
17 if (areValidationErrors(req, res)) return 18 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
18 19
19 return next() 20 return next()
20 } 21 }
diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts
index f58c0676c..9d303eee2 100644
--- a/server/middlewares/validators/config.ts
+++ b/server/middlewares/validators/config.ts
@@ -24,6 +24,7 @@ const customConfigUpdateValidator = [
24 body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'), 24 body('transcoding.resolutions.480p').isBoolean().withMessage('Should have a valid transcoding 480p resolution enabled boolean'),
25 body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), 25 body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'),
26 body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), 26 body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'),
27 body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
27 28
28 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 29 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
29 logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) 30 logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body })
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index e3f0f5963..c5400c8f5 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -11,3 +11,4 @@ export * from './video-blacklist'
11export * from './video-channels' 11export * from './video-channels'
12export * from './webfinger' 12export * from './webfinger'
13export * from './search' 13export * from './search'
14export * from './video-imports'
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index 00bde548c..d85611773 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -8,6 +8,7 @@ const SORTABLE_JOBS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.JOBS)
8const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES) 8const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES)
9const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS) 9const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS)
10const SORTABLE_VIDEOS_SEARCH_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS_SEARCH) 10const SORTABLE_VIDEOS_SEARCH_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS_SEARCH)
11const SORTABLE_VIDEO_IMPORTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_IMPORTS)
11const SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_COMMENT_THREADS) 12const SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_COMMENT_THREADS)
12const SORTABLE_BLACKLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.BLACKLISTS) 13const SORTABLE_BLACKLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.BLACKLISTS)
13const SORTABLE_VIDEO_CHANNELS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_CHANNELS) 14const SORTABLE_VIDEO_CHANNELS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_CHANNELS)
@@ -19,6 +20,7 @@ const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS)
19const jobsSortValidator = checkSort(SORTABLE_JOBS_COLUMNS) 20const jobsSortValidator = checkSort(SORTABLE_JOBS_COLUMNS)
20const videoAbusesSortValidator = checkSort(SORTABLE_VIDEO_ABUSES_COLUMNS) 21const videoAbusesSortValidator = checkSort(SORTABLE_VIDEO_ABUSES_COLUMNS)
21const videosSortValidator = checkSort(SORTABLE_VIDEOS_COLUMNS) 22const videosSortValidator = checkSort(SORTABLE_VIDEOS_COLUMNS)
23const videoImportsSortValidator = checkSort(SORTABLE_VIDEO_IMPORTS_COLUMNS)
22const videosSearchSortValidator = checkSort(SORTABLE_VIDEOS_SEARCH_COLUMNS) 24const videosSearchSortValidator = checkSort(SORTABLE_VIDEOS_SEARCH_COLUMNS)
23const videoCommentThreadsSortValidator = checkSort(SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS) 25const videoCommentThreadsSortValidator = checkSort(SORTABLE_VIDEO_COMMENT_THREADS_COLUMNS)
24const blacklistSortValidator = checkSort(SORTABLE_BLACKLISTS_COLUMNS) 26const blacklistSortValidator = checkSort(SORTABLE_BLACKLISTS_COLUMNS)
@@ -32,6 +34,7 @@ export {
32 usersSortValidator, 34 usersSortValidator,
33 videoAbusesSortValidator, 35 videoAbusesSortValidator,
34 videoChannelsSortValidator, 36 videoChannelsSortValidator,
37 videoImportsSortValidator,
35 videosSearchSortValidator, 38 videosSearchSortValidator,
36 videosSortValidator, 39 videosSortValidator,
37 blacklistSortValidator, 40 blacklistSortValidator,
diff --git a/server/middlewares/validators/video-captions.ts b/server/middlewares/validators/video-captions.ts
index b6d92d380..18d537bc4 100644
--- a/server/middlewares/validators/video-captions.ts
+++ b/server/middlewares/validators/video-captions.ts
@@ -7,6 +7,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
7import { UserRight } from '../../../shared' 7import { UserRight } from '../../../shared'
8import { logger } from '../../helpers/logger' 8import { logger } from '../../helpers/logger'
9import { isVideoCaptionExist, isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' 9import { isVideoCaptionExist, isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions'
10import { cleanUpReqFiles } from '../../helpers/utils'
10 11
11const addVideoCaptionValidator = [ 12const addVideoCaptionValidator = [
12 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), 13 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
@@ -20,12 +21,12 @@ const addVideoCaptionValidator = [
20 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 21 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
21 logger.debug('Checking addVideoCaption parameters', { parameters: req.body }) 22 logger.debug('Checking addVideoCaption parameters', { parameters: req.body })
22 23
23 if (areValidationErrors(req, res)) return 24 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
24 if (!await isVideoExist(req.params.videoId, res)) return 25 if (!await isVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req)
25 26
26 // Check if the user who did the request is able to update the video 27 // Check if the user who did the request is able to update the video
27 const user = res.locals.oauth.token.User 28 const user = res.locals.oauth.token.User
28 if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return 29 if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
29 30
30 return next() 31 return next()
31 } 32 }
diff --git a/server/middlewares/validators/video-channels.ts b/server/middlewares/validators/video-channels.ts
index 7f65f7290..143ce9582 100644
--- a/server/middlewares/validators/video-channels.ts
+++ b/server/middlewares/validators/video-channels.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import { body, param } from 'express-validator/check' 2import { body, param } from 'express-validator/check'
3import { UserRight } from '../../../shared' 3import { UserRight } from '../../../shared'
4import { isAccountIdExist, isAccountNameWithHostExist } from '../../helpers/custom-validators/accounts' 4import { isAccountNameWithHostExist } from '../../helpers/custom-validators/accounts'
5import { isIdOrUUIDValid } from '../../helpers/custom-validators/misc' 5import { isIdOrUUIDValid } from '../../helpers/custom-validators/misc'
6import { 6import {
7 isVideoChannelDescriptionValid, 7 isVideoChannelDescriptionValid,
@@ -13,8 +13,6 @@ import { logger } from '../../helpers/logger'
13import { UserModel } from '../../models/account/user' 13import { UserModel } from '../../models/account/user'
14import { VideoChannelModel } from '../../models/video/video-channel' 14import { VideoChannelModel } from '../../models/video/video-channel'
15import { areValidationErrors } from './utils' 15import { areValidationErrors } from './utils'
16import { isAvatarFile } from '../../helpers/custom-validators/users'
17import { CONSTRAINTS_FIELDS } from '../../initializers'
18 16
19const listVideoAccountChannelsValidator = [ 17const listVideoAccountChannelsValidator = [
20 param('accountName').exists().withMessage('Should have a valid account name'), 18 param('accountName').exists().withMessage('Should have a valid account name'),
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
new file mode 100644
index 000000000..d806edfa3
--- /dev/null
+++ b/server/middlewares/validators/video-imports.ts
@@ -0,0 +1,47 @@
1import * as express from 'express'
2import { body } from 'express-validator/check'
3import { isIdValid } from '../../helpers/custom-validators/misc'
4import { logger } from '../../helpers/logger'
5import { areValidationErrors } from './utils'
6import { getCommonVideoAttributes } from './videos'
7import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
8import { cleanUpReqFiles } from '../../helpers/utils'
9import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'
10import { CONFIG } from '../../initializers/constants'
11
12const videoImportAddValidator = getCommonVideoAttributes().concat([
13 body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
14 body('channelId')
15 .toInt()
16 .custom(isIdValid).withMessage('Should have correct video channel id'),
17 body('name')
18 .optional()
19 .custom(isVideoNameValid).withMessage('Should have a valid name'),
20
21 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
22 logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body })
23
24 const user = res.locals.oauth.token.User
25
26 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
27
28 if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
29 cleanUpReqFiles(req)
30 return res.status(409)
31 .json({ error: 'Import is not enabled on this instance.' })
32 .end()
33 }
34
35 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
36
37 return next()
38 }
39])
40
41// ---------------------------------------------------------------------------
42
43export {
44 videoImportAddValidator
45}
46
47// ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts
index d9af2aa0a..c812d4677 100644
--- a/server/middlewares/validators/videos.ts
+++ b/server/middlewares/validators/videos.ts
@@ -35,6 +35,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers'
35import { VideoShareModel } from '../../models/video/video-share' 35import { VideoShareModel } from '../../models/video/video-share'
36import { authenticate } from '../oauth' 36import { authenticate } from '../oauth'
37import { areValidationErrors } from './utils' 37import { areValidationErrors } from './utils'
38import { cleanUpReqFiles } from '../../helpers/utils'
38 39
39const videosAddValidator = getCommonVideoAttributes().concat([ 40const videosAddValidator = getCommonVideoAttributes().concat([
40 body('videofile') 41 body('videofile')
@@ -50,13 +51,13 @@ const videosAddValidator = getCommonVideoAttributes().concat([
50 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 51 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
51 logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) 52 logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files })
52 53
53 if (areValidationErrors(req, res)) return 54 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
54 if (areErrorsInScheduleUpdate(req, res)) return 55 if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
55 56
56 const videoFile: Express.Multer.File = req.files['videofile'][0] 57 const videoFile: Express.Multer.File = req.files['videofile'][0]
57 const user = res.locals.oauth.token.User 58 const user = res.locals.oauth.token.User
58 59
59 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return 60 if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
60 61
61 const isAble = await user.isAbleToUploadVideo(videoFile) 62 const isAble = await user.isAbleToUploadVideo(videoFile)
62 if (isAble === false) { 63 if (isAble === false) {
@@ -64,7 +65,7 @@ const videosAddValidator = getCommonVideoAttributes().concat([
64 .json({ error: 'The user video quota is exceeded with this video.' }) 65 .json({ error: 'The user video quota is exceeded with this video.' })
65 .end() 66 .end()
66 67
67 return 68 return cleanUpReqFiles(req)
68 } 69 }
69 70
70 let duration: number 71 let duration: number
@@ -77,7 +78,7 @@ const videosAddValidator = getCommonVideoAttributes().concat([
77 .json({ error: 'Invalid input file.' }) 78 .json({ error: 'Invalid input file.' })
78 .end() 79 .end()
79 80
80 return 81 return cleanUpReqFiles(req)
81 } 82 }
82 83
83 videoFile['duration'] = duration 84 videoFile['duration'] = duration
@@ -99,23 +100,24 @@ const videosUpdateValidator = getCommonVideoAttributes().concat([
99 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 100 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
100 logger.debug('Checking videosUpdate parameters', { parameters: req.body }) 101 logger.debug('Checking videosUpdate parameters', { parameters: req.body })
101 102
102 if (areValidationErrors(req, res)) return 103 if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
103 if (areErrorsInScheduleUpdate(req, res)) return 104 if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
104 if (!await isVideoExist(req.params.id, res)) return 105 if (!await isVideoExist(req.params.id, res)) return cleanUpReqFiles(req)
105 106
106 const video = res.locals.video 107 const video = res.locals.video
107 108
108 // Check if the user who did the request is able to update the video 109 // Check if the user who did the request is able to update the video
109 const user = res.locals.oauth.token.User 110 const user = res.locals.oauth.token.User
110 if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return 111 if (!checkUserCanManageVideo(user, res.locals.video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
111 112
112 if (video.privacy !== VideoPrivacy.PRIVATE && req.body.privacy === VideoPrivacy.PRIVATE) { 113 if (video.privacy !== VideoPrivacy.PRIVATE && req.body.privacy === VideoPrivacy.PRIVATE) {
114 cleanUpReqFiles(req)
113 return res.status(409) 115 return res.status(409)
114 .json({ error: 'Cannot set "private" a video that was not private.' }) 116 .json({ error: 'Cannot set "private" a video that was not private.' })
115 .end() 117 .end()
116 } 118 }
117 119
118 if (req.body.channelId && !await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return 120 if (req.body.channelId && !await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
119 121
120 return next() 122 return next()
121 } 123 }
@@ -221,36 +223,6 @@ const videosShareValidator = [
221 } 223 }
222] 224]
223 225
224// ---------------------------------------------------------------------------
225
226export {
227 videosAddValidator,
228 videosUpdateValidator,
229 videosGetValidator,
230 videosRemoveValidator,
231 videosShareValidator,
232
233 videoAbuseReportValidator,
234
235 videoRateValidator
236}
237
238// ---------------------------------------------------------------------------
239
240function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) {
241 if (req.body.scheduleUpdate) {
242 if (!req.body.scheduleUpdate.updateAt) {
243 res.status(400)
244 .json({ error: 'Schedule update at is mandatory.' })
245 .end()
246
247 return true
248 }
249 }
250
251 return false
252}
253
254function getCommonVideoAttributes () { 226function getCommonVideoAttributes () {
255 return [ 227 return [
256 body('thumbnailfile') 228 body('thumbnailfile')
@@ -317,3 +289,35 @@ function getCommonVideoAttributes () {
317 .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy') 289 .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy')
318 ] as (ValidationChain | express.Handler)[] 290 ] as (ValidationChain | express.Handler)[]
319} 291}
292
293// ---------------------------------------------------------------------------
294
295export {
296 videosAddValidator,
297 videosUpdateValidator,
298 videosGetValidator,
299 videosRemoveValidator,
300 videosShareValidator,
301
302 videoAbuseReportValidator,
303
304 videoRateValidator,
305
306 getCommonVideoAttributes
307}
308
309// ---------------------------------------------------------------------------
310
311function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) {
312 if (req.body.scheduleUpdate) {
313 if (!req.body.scheduleUpdate.updateAt) {
314 res.status(400)
315 .json({ error: 'Schedule update at is mandatory.' })
316 .end()
317
318 return true
319 }
320 }
321
322 return false
323}
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index d674d8d22..66f5dcf2e 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -16,7 +16,6 @@ import {
16} from 'sequelize-typescript' 16} from 'sequelize-typescript'
17import { Account } from '../../../shared/models/actors' 17import { Account } from '../../../shared/models/actors'
18import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' 18import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts'
19import { logger } from '../../helpers/logger'
20import { sendDeleteActor } from '../../lib/activitypub/send' 19import { sendDeleteActor } from '../../lib/activitypub/send'
21import { ActorModel } from '../activitypub/actor' 20import { ActorModel } from '../activitypub/actor'
22import { ApplicationModel } from '../application/application' 21import { ApplicationModel } from '../application/application'
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 267032e2a..35d7c35e8 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -91,6 +91,9 @@ enum ScopeNames {
91 fields: [ 'inboxUrl', 'sharedInboxUrl' ] 91 fields: [ 'inboxUrl', 'sharedInboxUrl' ]
92 }, 92 },
93 { 93 {
94 fields: [ 'sharedInboxUrl' ]
95 },
96 {
94 fields: [ 'serverId' ] 97 fields: [ 'serverId' ]
95 }, 98 },
96 { 99 {
@@ -454,6 +457,10 @@ export class ActorModel extends Model<ActorModel> {
454 return 'acct:' + this.preferredUsername + '@' + this.getHost() 457 return 'acct:' + this.preferredUsername + '@' + this.getHost()
455 } 458 }
456 459
460 getIdentifier () {
461 return this.Server ? `${this.preferredUsername}@${this.Server.host}` : this.preferredUsername
462 }
463
457 getHost () { 464 getHost () {
458 return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST 465 return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST
459 } 466 }
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
new file mode 100644
index 000000000..eca87163d
--- /dev/null
+++ b/server/models/video/video-import.ts
@@ -0,0 +1,175 @@
1import {
2 AfterUpdate,
3 AllowNull,
4 BelongsTo,
5 Column,
6 CreatedAt,
7 DataType,
8 Default,
9 DefaultScope,
10 ForeignKey,
11 Is,
12 Model,
13 Table,
14 UpdatedAt
15} from 'sequelize-typescript'
16import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
17import { getSort, throwIfNotValid } from '../utils'
18import { VideoModel } from './video'
19import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
20import { VideoImport, VideoImportState } from '../../../shared'
21import { VideoChannelModel } from './video-channel'
22import { AccountModel } from '../account/account'
23import { TagModel } from './tag'
24
25@DefaultScope({
26 include: [
27 {
28 model: () => VideoModel,
29 required: false,
30 include: [
31 {
32 model: () => VideoChannelModel,
33 required: true,
34 include: [
35 {
36 model: () => AccountModel,
37 required: true
38 }
39 ]
40 },
41 {
42 model: () => TagModel
43 }
44 ]
45 }
46 ]
47})
48
49@Table({
50 tableName: 'videoImport',
51 indexes: [
52 {
53 fields: [ 'videoId' ],
54 unique: true
55 }
56 ]
57})
58export class VideoImportModel extends Model<VideoImportModel> {
59 @CreatedAt
60 createdAt: Date
61
62 @UpdatedAt
63 updatedAt: Date
64
65 @AllowNull(false)
66 @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
67 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
68 targetUrl: string
69
70 @AllowNull(false)
71 @Default(null)
72 @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))
73 @Column
74 state: VideoImportState
75
76 @AllowNull(true)
77 @Default(null)
78 @Column(DataType.TEXT)
79 error: string
80
81 @ForeignKey(() => VideoModel)
82 @Column
83 videoId: number
84
85 @BelongsTo(() => VideoModel, {
86 foreignKey: {
87 allowNull: true
88 },
89 onDelete: 'set null'
90 })
91 Video: VideoModel
92
93 @AfterUpdate
94 static deleteVideoIfFailed (instance: VideoImportModel, options) {
95 if (instance.state === VideoImportState.FAILED) {
96 return instance.Video.destroy({ transaction: options.transaction })
97 }
98
99 return undefined
100 }
101
102 static loadAndPopulateVideo (id: number) {
103 return VideoImportModel.findById(id)
104 }
105
106 static listUserVideoImportsForApi (accountId: number, start: number, count: number, sort: string) {
107 const query = {
108 distinct: true,
109 offset: start,
110 limit: count,
111 order: getSort(sort),
112 include: [
113 {
114 model: VideoModel,
115 required: false,
116 include: [
117 {
118 model: VideoChannelModel,
119 required: true,
120 include: [
121 {
122 model: AccountModel,
123 required: true,
124 where: {
125 id: accountId
126 }
127 }
128 ]
129 },
130 {
131 model: TagModel,
132 required: false
133 }
134 ]
135 }
136 ]
137 }
138
139 return VideoImportModel.unscoped()
140 .findAndCountAll(query)
141 .then(({ rows, count }) => {
142 return {
143 data: rows,
144 total: count
145 }
146 })
147 }
148
149 toFormattedJSON (): VideoImport {
150 const videoFormatOptions = {
151 additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true }
152 }
153 const video = this.Video
154 ? Object.assign(this.Video.toFormattedJSON(videoFormatOptions), {
155 tags: this.Video.Tags.map(t => t.name)
156 })
157 : undefined
158
159 return {
160 id: this.id,
161 targetUrl: this.targetUrl,
162 state: {
163 id: this.state,
164 label: VideoImportModel.getStateLabel(this.state)
165 },
166 error: this.error,
167 updatedAt: this.updatedAt.toISOString(),
168 createdAt: this.createdAt.toISOString(),
169 video
170 }
171 }
172 private static getStateLabel (id: number) {
173 return VIDEO_IMPORT_STATES[id] || 'Unknown'
174 }
175}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index a6c4620b2..39fe21007 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -377,7 +377,7 @@ type AvailableForListOptions = {
377 include: [ 377 include: [
378 { 378 {
379 model: () => VideoFileModel.unscoped(), 379 model: () => VideoFileModel.unscoped(),
380 required: true 380 required: false
381 } 381 }
382 ] 382 ]
383 }, 383 },
@@ -957,8 +957,10 @@ export class VideoModel extends Model<VideoModel> {
957 }) 957 })
958 } 958 }
959 959
960 static load (id: number) { 960 static load (id: number, t?: Sequelize.Transaction) {
961 return VideoModel.findById(id) 961 const options = t ? { transaction: t } : undefined
962
963 return VideoModel.findById(id, options)
962 } 964 }
963 965
964 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { 966 static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
@@ -1353,7 +1355,8 @@ export class VideoModel extends Model<VideoModel> {
1353 mimeType: VIDEO_EXT_MIMETYPE[file.extname], 1355 mimeType: VIDEO_EXT_MIMETYPE[file.extname],
1354 href: this.getVideoFileUrl(file, baseUrlHttp), 1356 href: this.getVideoFileUrl(file, baseUrlHttp),
1355 width: file.resolution, 1357 width: file.resolution,
1356 size: file.size 1358 size: file.size,
1359 fps: file.fps
1357 }) 1360 })
1358 1361
1359 url.push({ 1362 url.push({
@@ -1569,21 +1572,25 @@ export class VideoModel extends Model<VideoModel> {
1569 removeThumbnail () { 1572 removeThumbnail () {
1570 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) 1573 const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName())
1571 return unlinkPromise(thumbnailPath) 1574 return unlinkPromise(thumbnailPath)
1575 .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err }))
1572 } 1576 }
1573 1577
1574 removePreview () { 1578 removePreview () {
1575 // Same name than video thumbnail 1579 const previewPath = join(CONFIG.STORAGE.PREVIEWS_DIR + this.getPreviewName())
1576 return unlinkPromise(CONFIG.STORAGE.PREVIEWS_DIR + this.getPreviewName()) 1580 return unlinkPromise(previewPath)
1581 .catch(err => logger.warn('Cannot delete preview %s.', previewPath, { err }))
1577 } 1582 }
1578 1583
1579 removeFile (videoFile: VideoFileModel) { 1584 removeFile (videoFile: VideoFileModel) {
1580 const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile)) 1585 const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, this.getVideoFilename(videoFile))
1581 return unlinkPromise(filePath) 1586 return unlinkPromise(filePath)
1587 .catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
1582 } 1588 }
1583 1589
1584 removeTorrent (videoFile: VideoFileModel) { 1590 removeTorrent (videoFile: VideoFileModel) {
1585 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile)) 1591 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, this.getTorrentFileName(videoFile))
1586 return unlinkPromise(torrentPath) 1592 return unlinkPromise(torrentPath)
1593 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
1587 } 1594 }
1588 1595
1589 getActivityStreamDuration () { 1596 getActivityStreamDuration () {
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 03855237f..2742e26de 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -60,6 +60,13 @@ describe('Test config API validators', function () {
60 '720p': false, 60 '720p': false,
61 '1080p': false 61 '1080p': false
62 } 62 }
63 },
64 import: {
65 videos: {
66 http: {
67 enabled: false
68 }
69 }
63 } 70 }
64 } 71 }
65 72
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 820dde889..03fdd5c4e 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -10,4 +10,5 @@ import './video-captions'
10import './video-channels' 10import './video-channels'
11import './video-comments' 11import './video-comments'
12import './videos' 12import './videos'
13import './video-imports'
13import './search' 14import './search'
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts
new file mode 100644
index 000000000..0ead34a47
--- /dev/null
+++ b/server/tests/api/check-params/video-imports.ts
@@ -0,0 +1,275 @@
1/* tslint:disable:no-unused-expression */
2
3import { omit } from 'lodash'
4import 'mocha'
5import { join } from 'path'
6import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum'
7import {
8 createUser,
9 flushTests,
10 getMyUserInformation,
11 immutableAssign,
12 killallServers,
13 makeGetRequest,
14 makePostBodyRequest,
15 makeUploadRequest,
16 runServer,
17 ServerInfo,
18 setAccessTokensToServers,
19 updateCustomSubConfig,
20 userLogin
21} from '../../utils'
22import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../utils/requests/check-api-params'
23import { getYoutubeVideoUrl } from '../../utils/videos/video-imports'
24
25describe('Test video imports API validator', function () {
26 const path = '/api/v1/videos/imports'
27 let server: ServerInfo
28 let userAccessToken = ''
29 let accountName: string
30 let channelId: number
31 let channelUUID: string
32
33 // ---------------------------------------------------------------
34
35 before(async function () {
36 this.timeout(30000)
37
38 await flushTests()
39
40 server = await runServer(1)
41
42 await setAccessTokensToServers([ server ])
43
44 const username = 'user1'
45 const password = 'my super password'
46 await createUser(server.url, server.accessToken, username, password)
47 userAccessToken = await userLogin(server, { username, password })
48
49 {
50 const res = await getMyUserInformation(server.url, server.accessToken)
51 channelId = res.body.videoChannels[ 0 ].id
52 channelUUID = res.body.videoChannels[ 0 ].uuid
53 accountName = res.body.account.name + '@' + res.body.account.host
54 }
55 })
56
57 describe('When listing my video imports', function () {
58 const myPath = '/api/v1/users/me/videos/imports'
59
60 it('Should fail with a bad start pagination', async function () {
61 await checkBadStartPagination(server.url, myPath, server.accessToken)
62 })
63
64 it('Should fail with a bad count pagination', async function () {
65 await checkBadCountPagination(server.url, myPath, server.accessToken)
66 })
67
68 it('Should fail with an incorrect sort', async function () {
69 await checkBadSortPagination(server.url, myPath, server.accessToken)
70 })
71
72 it('Should success with the correct parameters', async function () {
73 await makeGetRequest({ url: server.url, path: myPath, statusCodeExpected: 200, token: server.accessToken })
74 })
75 })
76
77 describe('When adding a video import', function () {
78 let baseCorrectParams
79
80 before(function () {
81 baseCorrectParams = {
82 targetUrl: getYoutubeVideoUrl(),
83 name: 'my super name',
84 category: 5,
85 licence: 1,
86 language: 'pt',
87 nsfw: false,
88 commentsEnabled: true,
89 waitTranscoding: true,
90 description: 'my super description',
91 support: 'my super support text',
92 tags: [ 'tag1', 'tag2' ],
93 privacy: VideoPrivacy.PUBLIC,
94 channelId: channelId
95 }
96 })
97
98 it('Should fail with nothing', async function () {
99 const fields = {}
100 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
101 })
102
103 it('Should fail without a target url', async function () {
104 const fields = omit(baseCorrectParams, 'targetUrl')
105 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields, statusCodeExpected: 400 })
106 })
107
108 it('Should fail with a bad target url', async function () {
109 const fields = immutableAssign(baseCorrectParams, { targetUrl: 'htt://hello' })
110
111 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
112 })
113
114 it('Should fail with a long name', async function () {
115 const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) })
116
117 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
118 })
119
120 it('Should fail with a bad category', async function () {
121 const fields = immutableAssign(baseCorrectParams, { category: 125 })
122
123 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
124 })
125
126 it('Should fail with a bad licence', async function () {
127 const fields = immutableAssign(baseCorrectParams, { licence: 125 })
128
129 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
130 })
131
132 it('Should fail with a bad language', async function () {
133 const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) })
134
135 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
136 })
137
138 it('Should fail with a long description', async function () {
139 const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) })
140
141 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
142 })
143
144 it('Should fail with a long support text', async function () {
145 const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(150) })
146
147 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
148 })
149
150 it('Should fail without a channel', async function () {
151 const fields = omit(baseCorrectParams, 'channelId')
152
153 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
154 })
155
156 it('Should fail with a bad channel', async function () {
157 const fields = immutableAssign(baseCorrectParams, { channelId: 545454 })
158
159 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
160 })
161
162 it('Should fail with another user channel', async function () {
163 const user = {
164 username: 'fake',
165 password: 'fake_password'
166 }
167 await createUser(server.url, server.accessToken, user.username, user.password)
168
169 const accessTokenUser = await userLogin(server, user)
170 const res = await getMyUserInformation(server.url, accessTokenUser)
171 const customChannelId = res.body.videoChannels[0].id
172
173 const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId })
174
175 await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields })
176 })
177
178 it('Should fail with too many tags', async function () {
179 const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] })
180
181 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
182 })
183
184 it('Should fail with a tag length too low', async function () {
185 const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] })
186
187 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
188 })
189
190 it('Should fail with a tag length too big', async function () {
191 const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] })
192
193 await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
194 })
195
196 it('Should fail with an incorrect thumbnail file', async function () {
197 const fields = baseCorrectParams
198 const attaches = {
199 'thumbnailfile': join(__dirname, '..', '..', 'fixtures', 'avatar.png')
200 }
201
202 await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
203 })
204
205 it('Should fail with a big thumbnail file', async function () {
206 const fields = baseCorrectParams
207 const attaches = {
208 'thumbnailfile': join(__dirname, '..', '..', 'fixtures', 'avatar-big.png')
209 }
210
211 await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
212 })
213
214 it('Should fail with an incorrect preview file', async function () {
215 const fields = baseCorrectParams
216 const attaches = {
217 'previewfile': join(__dirname, '..', '..', 'fixtures', 'avatar.png')
218 }
219
220 await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
221 })
222
223 it('Should fail with a big preview file', async function () {
224 const fields = baseCorrectParams
225 const attaches = {
226 'previewfile': join(__dirname, '..', '..', 'fixtures', 'avatar-big.png')
227 }
228
229 await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
230 })
231
232 it('Should succeed with the correct parameters', async function () {
233 this.timeout(10000)
234
235 {
236 await makePostBodyRequest({
237 url: server.url,
238 path,
239 token: server.accessToken,
240 fields: baseCorrectParams,
241 statusCodeExpected: 200
242 })
243 }
244 })
245
246 it('Should forbid to import videos', async function () {
247 await updateCustomSubConfig(server.url, server.accessToken, {
248 import: {
249 videos: {
250 http: {
251 enabled: false
252 }
253 }
254 }
255 })
256
257 await makePostBodyRequest({
258 url: server.url,
259 path,
260 token: server.accessToken,
261 fields: baseCorrectParams,
262 statusCodeExpected: 409
263 })
264 })
265 })
266
267 after(async function () {
268 killallServers([ server ])
269
270 // Keep the logs if the test failed
271 if (this['ok']) {
272 await flushTests()
273 }
274 })
275})
diff --git a/server/tests/api/index-slow.ts b/server/tests/api/index-slow.ts
index d987442b3..243c941cb 100644
--- a/server/tests/api/index-slow.ts
+++ b/server/tests/api/index-slow.ts
@@ -1,4 +1,5 @@
1// Order of the tests we want to execute 1// Order of the tests we want to execute
2import './videos/video-channels'
2import './videos/video-transcoder' 3import './videos/video-transcoder'
3import './videos/multiple-servers' 4import './videos/multiple-servers'
4import './server/follows' 5import './server/follows'
@@ -7,3 +8,4 @@ import './videos/video-comments'
7import './users/users-multiple-servers' 8import './users/users-multiple-servers'
8import './server/handle-down' 9import './server/handle-down'
9import './videos/video-schedule-update' 10import './videos/video-schedule-update'
11import './videos/video-imports'
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index 1782a8623..b65061a5d 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -44,6 +44,7 @@ function checkInitialConfig (data: CustomConfig) {
44 expect(data.transcoding.resolutions['480p']).to.be.true 44 expect(data.transcoding.resolutions['480p']).to.be.true
45 expect(data.transcoding.resolutions['720p']).to.be.true 45 expect(data.transcoding.resolutions['720p']).to.be.true
46 expect(data.transcoding.resolutions['1080p']).to.be.true 46 expect(data.transcoding.resolutions['1080p']).to.be.true
47 expect(data.import.videos.http.enabled).to.be.true
47} 48}
48 49
49function checkUpdatedConfig (data: CustomConfig) { 50function checkUpdatedConfig (data: CustomConfig) {
@@ -70,6 +71,7 @@ function checkUpdatedConfig (data: CustomConfig) {
70 expect(data.transcoding.resolutions['480p']).to.be.true 71 expect(data.transcoding.resolutions['480p']).to.be.true
71 expect(data.transcoding.resolutions['720p']).to.be.false 72 expect(data.transcoding.resolutions['720p']).to.be.false
72 expect(data.transcoding.resolutions['1080p']).to.be.false 73 expect(data.transcoding.resolutions['1080p']).to.be.false
74 expect(data.import.videos.http.enabled).to.be.false
73} 75}
74 76
75describe('Test config', function () { 77describe('Test config', function () {
@@ -160,6 +162,13 @@ describe('Test config', function () {
160 '720p': false, 162 '720p': false,
161 '1080p': false 163 '1080p': false
162 } 164 }
165 },
166 import: {
167 videos: {
168 http: {
169 enabled: false
170 }
171 }
163 } 172 }
164 } 173 }
165 await updateCustomConfig(server.url, server.accessToken, newCustomConfig) 174 await updateCustomConfig(server.url, server.accessToken, newCustomConfig)
diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts
index a19b47509..25c87b4dc 100644
--- a/server/tests/api/server/follows.ts
+++ b/server/tests/api/server/follows.ts
@@ -178,7 +178,7 @@ describe('Test follows', function () {
178 }) 178 })
179 179
180 it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () { 180 it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () {
181 this.timeout(10000) 181 this.timeout(35000)
182 182
183 await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'server2' }) 183 await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'server2' })
184 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3' }) 184 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3' })
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index fb9722630..18a0d9ce3 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -176,7 +176,7 @@ describe('Test handle downs', function () {
176 }) 176 })
177 177
178 it('Should re-follow server 1', async function () { 178 it('Should re-follow server 1', async function () {
179 this.timeout(15000) 179 this.timeout(35000)
180 180
181 await reRunServer(servers[1]) 181 await reRunServer(servers[1])
182 await reRunServer(servers[2]) 182 await reRunServer(servers[2])
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index cb18898ce..516dc5aba 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -209,19 +209,19 @@ describe('Test multiple servers', function () {
209 files: [ 209 files: [
210 { 210 {
211 resolution: 240, 211 resolution: 240,
212 size: 190000 212 size: 280000
213 }, 213 },
214 { 214 {
215 resolution: 360, 215 resolution: 360,
216 size: 280000 216 size: 370000
217 }, 217 },
218 { 218 {
219 resolution: 480, 219 resolution: 480,
220 size: 390000 220 size: 470000
221 }, 221 },
222 { 222 {
223 resolution: 720, 223 resolution: 720,
224 size: 710000 224 size: 740000
225 } 225 }
226 ], 226 ],
227 thumbnailfile: 'thumbnail', 227 thumbnailfile: 'thumbnail',
@@ -975,19 +975,19 @@ describe('Test multiple servers', function () {
975 files: [ 975 files: [
976 { 976 {
977 resolution: 720, 977 resolution: 720,
978 size: 40315 978 size: 36000
979 }, 979 },
980 { 980 {
981 resolution: 480, 981 resolution: 480,
982 size: 22808 982 size: 21000
983 }, 983 },
984 { 984 {
985 resolution: 360, 985 resolution: 360,
986 size: 18617 986 size: 17000
987 }, 987 },
988 { 988 {
989 resolution: 240, 989 resolution: 240,
990 size: 15217 990 size: 13000
991 } 991 }
992 ] 992 ]
993 } 993 }
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
new file mode 100644
index 000000000..f21ade5c3
--- /dev/null
+++ b/server/tests/api/videos/video-imports.ts
@@ -0,0 +1,161 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos'
6import {
7 doubleFollow,
8 flushAndRunMultipleServers,
9 getMyUserInformation,
10 getMyVideos,
11 getVideo,
12 getVideosList,
13 killallServers,
14 ServerInfo,
15 setAccessTokensToServers
16} from '../../utils'
17import { waitJobs } from '../../utils/server/jobs'
18import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../utils/videos/video-imports'
19
20const expect = chai.expect
21
22describe('Test video imports', function () {
23 let servers: ServerInfo[] = []
24 let channelIdServer1: number
25 let channelIdServer2: number
26
27 async function checkVideoServer1 (url: string, id: number | string) {
28 const res = await getVideo(url, id)
29 const video: VideoDetails = res.body
30
31 expect(video.name).to.equal('small video - youtube')
32 expect(video.category.label).to.equal('News')
33 expect(video.licence.label).to.equal('Attribution')
34 expect(video.language.label).to.equal('Unknown')
35 expect(video.nsfw).to.be.false
36 expect(video.description).to.equal('this is a super description')
37 expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
38
39 expect(video.files).to.have.lengthOf(1)
40 }
41
42 async function checkVideoServer2 (url: string, id: number | string) {
43 const res = await getVideo(url, id)
44 const video = res.body
45
46 expect(video.name).to.equal('my super name')
47 expect(video.category.label).to.equal('Entertainment')
48 expect(video.licence.label).to.equal('Public Domain Dedication')
49 expect(video.language.label).to.equal('English')
50 expect(video.nsfw).to.be.false
51 expect(video.description).to.equal('my super description')
52 expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
53
54 expect(video.files).to.have.lengthOf(1)
55 }
56
57 before(async function () {
58 this.timeout(30000)
59
60 // Run servers
61 servers = await flushAndRunMultipleServers(2)
62
63 await setAccessTokensToServers(servers)
64
65 {
66 const res = await getMyUserInformation(servers[0].url, servers[0].accessToken)
67 channelIdServer1 = res.body.videoChannels[ 0 ].id
68 }
69
70 {
71 const res = await getMyUserInformation(servers[1].url, servers[1].accessToken)
72 channelIdServer2 = res.body.videoChannels[ 0 ].id
73 }
74
75 await doubleFollow(servers[0], servers[1])
76 })
77
78 it('Should import a video on server 1', async function () {
79 this.timeout(60000)
80
81 const attributes = {
82 targetUrl: getYoutubeVideoUrl(),
83 channelId: channelIdServer1,
84 privacy: VideoPrivacy.PUBLIC
85 }
86 const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)
87 expect(res.body.video.name).to.equal('small video - youtube')
88 })
89
90 it('Should list the video to import in my videos on server 1', async function () {
91 const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5)
92
93 expect(res.body.total).to.equal(1)
94
95 const videos = res.body.data
96 expect(videos).to.have.lengthOf(1)
97 expect(videos[0].name).to.equal('small video - youtube')
98 })
99
100 it('Should list the video to import in my imports on server 1', async function () {
101 const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
102
103 expect(res.body.total).to.equal(1)
104 const videoImports = res.body.data
105 expect(videoImports).to.have.lengthOf(1)
106
107 expect(videoImports[0].targetUrl).to.equal(getYoutubeVideoUrl())
108 expect(videoImports[0].video.name).to.equal('small video - youtube')
109 })
110
111 it('Should have the video listed on the two instances1', async function () {
112 this.timeout(120000)
113
114 await waitJobs(servers)
115
116 for (const server of servers) {
117 const res = await getVideosList(server.url)
118 expect(res.body.total).to.equal(1)
119 expect(res.body.data).to.have.lengthOf(1)
120
121 await checkVideoServer1(server.url, res.body.data[0].uuid)
122 }
123 })
124
125 it('Should import a video on server 2 with some fields', async function () {
126 this.timeout(60000)
127
128 const attributes = {
129 targetUrl: getYoutubeVideoUrl(),
130 channelId: channelIdServer1,
131 privacy: VideoPrivacy.PUBLIC,
132 category: 10,
133 licence: 7,
134 language: 'en',
135 name: 'my super name',
136 description: 'my super description',
137 tags: [ 'supertag1', 'supertag2' ]
138 }
139 const res = await importVideo(servers[1].url, servers[1].accessToken, attributes)
140 expect(res.body.video.name).to.equal('my super name')
141 })
142
143 it('Should have the video listed on the two instances', async function () {
144 this.timeout(120000)
145
146 await waitJobs(servers)
147
148 for (const server of servers) {
149 const res = await getVideosList(server.url)
150 expect(res.body.total).to.equal(2)
151 expect(res.body.data).to.have.lengthOf(2)
152
153 await checkVideoServer2(server.url, res.body.data[0].uuid)
154 await checkVideoServer1(server.url, res.body.data[1].uuid)
155 }
156 })
157
158 after(async function () {
159 killallServers(servers)
160 })
161})
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index fe750253e..0f83d4d57 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -2,9 +2,12 @@
2 2
3import * as chai from 'chai' 3import * as chai from 'chai'
4import 'mocha' 4import 'mocha'
5import { omit } from 'lodash'
6import * as ffmpeg from 'fluent-ffmpeg'
5import { VideoDetails, VideoState } from '../../../../shared/models/videos' 7import { VideoDetails, VideoState } from '../../../../shared/models/videos'
6import { getVideoFileFPS } from '../../../helpers/ffmpeg-utils' 8import { getVideoFileFPS, audio } from '../../../helpers/ffmpeg-utils'
7import { 9import {
10 buildAbsoluteFixturePath,
8 doubleFollow, 11 doubleFollow,
9 flushAndRunMultipleServers, 12 flushAndRunMultipleServers,
10 getMyVideos, 13 getMyVideos,
@@ -32,6 +35,8 @@ describe('Test video transcoding', function () {
32 servers = await flushAndRunMultipleServers(2) 35 servers = await flushAndRunMultipleServers(2)
33 36
34 await setAccessTokensToServers(servers) 37 await setAccessTokensToServers(servers)
38
39 await doubleFollow(servers[0], servers[1])
35 }) 40 })
36 41
37 it('Should not transcode video on server 1', async function () { 42 it('Should not transcode video on server 1', async function () {
@@ -46,20 +51,22 @@ describe('Test video transcoding', function () {
46 51
47 await waitJobs(servers) 52 await waitJobs(servers)
48 53
49 const res = await getVideosList(servers[0].url) 54 for (const server of servers) {
50 const video = res.body.data[0] 55 const res = await getVideosList(server.url)
56 const video = res.body.data[ 0 ]
51 57
52 const res2 = await getVideo(servers[0].url, video.id) 58 const res2 = await getVideo(server.url, video.id)
53 const videoDetails = res2.body 59 const videoDetails = res2.body
54 expect(videoDetails.files).to.have.lengthOf(1) 60 expect(videoDetails.files).to.have.lengthOf(1)
55 61
56 const magnetUri = videoDetails.files[0].magnetUri 62 const magnetUri = videoDetails.files[ 0 ].magnetUri
57 expect(magnetUri).to.match(/\.webm/) 63 expect(magnetUri).to.match(/\.webm/)
58 64
59 const torrent = await webtorrentAdd(magnetUri) 65 const torrent = await webtorrentAdd(magnetUri, true)
60 expect(torrent.files).to.be.an('array') 66 expect(torrent.files).to.be.an('array')
61 expect(torrent.files.length).to.equal(1) 67 expect(torrent.files.length).to.equal(1)
62 expect(torrent.files[0].path).match(/\.webm$/) 68 expect(torrent.files[ 0 ].path).match(/\.webm$/)
69 }
63 }) 70 })
64 71
65 it('Should transcode video on server 2', async function () { 72 it('Should transcode video on server 2', async function () {
@@ -74,21 +81,112 @@ describe('Test video transcoding', function () {
74 81
75 await waitJobs(servers) 82 await waitJobs(servers)
76 83
77 const res = await getVideosList(servers[1].url) 84 for (const server of servers) {
85 const res = await getVideosList(server.url)
86
87 const video = res.body.data.find(v => v.name === videoAttributes.name)
88 const res2 = await getVideo(server.url, video.id)
89 const videoDetails = res2.body
90
91 expect(videoDetails.files).to.have.lengthOf(4)
92
93 const magnetUri = videoDetails.files[ 0 ].magnetUri
94 expect(magnetUri).to.match(/\.mp4/)
95
96 const torrent = await webtorrentAdd(magnetUri, true)
97 expect(torrent.files).to.be.an('array')
98 expect(torrent.files.length).to.equal(1)
99 expect(torrent.files[ 0 ].path).match(/\.mp4$/)
100 }
101 })
102
103 it('Should transcode high bit rate mp3 to proper bit rate', async function () {
104 this.timeout(60000)
105
106 const videoAttributes = {
107 name: 'mp3_256k',
108 fixture: 'video_short_mp3_256k.mp4'
109 }
110 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
111
112 await waitJobs(servers)
113
114 for (const server of servers) {
115 const res = await getVideosList(server.url)
116
117 const video = res.body.data.find(v => v.name === videoAttributes.name)
118 const res2 = await getVideo(server.url, video.id)
119 const videoDetails: VideoDetails = res2.body
120
121 expect(videoDetails.files).to.have.lengthOf(4)
122
123 const path = join(root(), 'test2', 'videos', video.uuid + '-240.mp4')
124 const probe = await audio.get(ffmpeg, path)
125
126 if (probe.audioStream) {
127 expect(probe.audioStream[ 'codec_name' ]).to.be.equal('aac')
128 expect(probe.audioStream[ 'bit_rate' ]).to.be.at.most(384 * 8000)
129 } else {
130 this.fail('Could not retrieve the audio stream on ' + probe.absolutePath)
131 }
132 }
133 })
134
135 it('Should transcode video with no audio and have no audio itself', async function () {
136 this.timeout(60000)
137
138 const videoAttributes = {
139 name: 'no_audio',
140 fixture: 'video_short_no_audio.mp4'
141 }
142 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
143
144 await waitJobs(servers)
145
146 for (const server of servers) {
147 const res = await getVideosList(server.url)
148
149 const video = res.body.data.find(v => v.name === videoAttributes.name)
150 const res2 = await getVideo(server.url, video.id)
151 const videoDetails: VideoDetails = res2.body
152
153 expect(videoDetails.files).to.have.lengthOf(4)
154 const path = join(root(), 'test2', 'videos', video.uuid + '-240.mp4')
155 const probe = await audio.get(ffmpeg, path)
156 expect(probe).to.not.have.property('audioStream')
157 }
158 })
159
160 it('Should leave the audio untouched, but properly transcode the video', async function () {
161 this.timeout(60000)
162
163 const videoAttributes = {
164 name: 'untouched_audio',
165 fixture: 'video_short.mp4'
166 }
167 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
78 168
79 const video = res.body.data[0] 169 await waitJobs(servers)
80 const res2 = await getVideo(servers[1].url, video.id)
81 const videoDetails = res2.body
82 170
83 expect(videoDetails.files).to.have.lengthOf(4) 171 for (const server of servers) {
172 const res = await getVideosList(server.url)
84 173
85 const magnetUri = videoDetails.files[0].magnetUri 174 const video = res.body.data.find(v => v.name === videoAttributes.name)
86 expect(magnetUri).to.match(/\.mp4/) 175 const res2 = await getVideo(server.url, video.id)
176 const videoDetails: VideoDetails = res2.body
87 177
88 const torrent = await webtorrentAdd(magnetUri) 178 expect(videoDetails.files).to.have.lengthOf(4)
89 expect(torrent.files).to.be.an('array') 179 const fixturePath = buildAbsoluteFixturePath(videoAttributes.fixture)
90 expect(torrent.files.length).to.equal(1) 180 const fixtureVideoProbe = await audio.get(ffmpeg, fixturePath)
91 expect(torrent.files[0].path).match(/\.mp4$/) 181 const path = join(root(), 'test2', 'videos', video.uuid + '-240.mp4')
182 const videoProbe = await audio.get(ffmpeg, path)
183 if (videoProbe.audioStream && fixtureVideoProbe.audioStream) {
184 const toOmit = [ 'max_bit_rate', 'duration', 'duration_ts', 'nb_frames', 'start_time', 'start_pts' ]
185 expect(omit(videoProbe.audioStream, toOmit)).to.be.deep.equal(omit(fixtureVideoProbe.audioStream, toOmit))
186 } else {
187 this.fail('Could not retrieve the audio stream on ' + videoProbe.absolutePath)
188 }
189 }
92 }) 190 })
93 191
94 it('Should transcode a 60 FPS video', async function () { 192 it('Should transcode a 60 FPS video', async function () {
@@ -103,38 +201,36 @@ describe('Test video transcoding', function () {
103 201
104 await waitJobs(servers) 202 await waitJobs(servers)
105 203
106 const res = await getVideosList(servers[1].url) 204 for (const server of servers) {
205 const res = await getVideosList(server.url)
206
207 const video = res.body.data.find(v => v.name === videoAttributes.name)
208 const res2 = await getVideo(server.url, video.id)
209 const videoDetails: VideoDetails = res2.body
210
211 expect(videoDetails.files).to.have.lengthOf(4)
212 expect(videoDetails.files[ 0 ].fps).to.be.above(58).and.below(62)
213 expect(videoDetails.files[ 1 ].fps).to.be.below(31)
214 expect(videoDetails.files[ 2 ].fps).to.be.below(31)
215 expect(videoDetails.files[ 3 ].fps).to.be.below(31)
107 216
108 const video = res.body.data[0] 217 for (const resolution of [ '240', '360', '480' ]) {
109 const res2 = await getVideo(servers[1].url, video.id) 218 const path = join(root(), 'test2', 'videos', video.uuid + '-' + resolution + '.mp4')
110 const videoDetails: VideoDetails = res2.body 219 const fps = await getVideoFileFPS(path)
111 220
112 expect(videoDetails.files).to.have.lengthOf(4) 221 expect(fps).to.be.below(31)
113 expect(videoDetails.files[0].fps).to.be.above(58).and.below(62) 222 }
114 expect(videoDetails.files[1].fps).to.be.below(31)
115 expect(videoDetails.files[2].fps).to.be.below(31)
116 expect(videoDetails.files[3].fps).to.be.below(31)
117 223
118 for (const resolution of [ '240', '360', '480' ]) { 224 const path = join(root(), 'test2', 'videos', video.uuid + '-720.mp4')
119 const path = join(root(), 'test2', 'videos', video.uuid + '-' + resolution + '.mp4')
120 const fps = await getVideoFileFPS(path) 225 const fps = await getVideoFileFPS(path)
121 226
122 expect(fps).to.be.below(31) 227 expect(fps).to.be.above(58).and.below(62)
123 } 228 }
124
125 const path = join(root(), 'test2', 'videos', video.uuid + '-720.mp4')
126 const fps = await getVideoFileFPS(path)
127
128 expect(fps).to.be.above(58).and.below(62)
129 }) 229 })
130 230
131 it('Should wait transcoding before publishing the video', async function () { 231 it('Should wait transcoding before publishing the video', async function () {
132 this.timeout(80000) 232 this.timeout(80000)
133 233
134 await doubleFollow(servers[0], servers[1])
135
136 await waitJobs(servers)
137
138 { 234 {
139 // Upload the video, but wait transcoding 235 // Upload the video, but wait transcoding
140 const videoAttributes = { 236 const videoAttributes = {
@@ -154,7 +250,7 @@ describe('Test video transcoding', function () {
154 250
155 // Should have my video 251 // Should have my video
156 const resMyVideos = await getMyVideos(servers[1].url, servers[1].accessToken, 0, 10) 252 const resMyVideos = await getMyVideos(servers[1].url, servers[1].accessToken, 0, 10)
157 const videoToFindInMine = resMyVideos.body.data.find(v => v.name === 'waiting video') 253 const videoToFindInMine = resMyVideos.body.data.find(v => v.name === videoAttributes.name)
158 expect(videoToFindInMine).not.to.be.undefined 254 expect(videoToFindInMine).not.to.be.undefined
159 expect(videoToFindInMine.state.id).to.equal(VideoState.TO_TRANSCODE) 255 expect(videoToFindInMine.state.id).to.equal(VideoState.TO_TRANSCODE)
160 expect(videoToFindInMine.state.label).to.equal('To transcode') 256 expect(videoToFindInMine.state.label).to.equal('To transcode')
@@ -162,7 +258,7 @@ describe('Test video transcoding', function () {
162 258
163 // Should not list this video 259 // Should not list this video
164 const resVideos = await getVideosList(servers[1].url) 260 const resVideos = await getVideosList(servers[1].url)
165 const videoToFindInList = resVideos.body.data.find(v => v.name === 'waiting video') 261 const videoToFindInList = resVideos.body.data.find(v => v.name === videoAttributes.name)
166 expect(videoToFindInList).to.be.undefined 262 expect(videoToFindInList).to.be.undefined
167 263
168 // Server 1 should not have the video yet 264 // Server 1 should not have the video yet
diff --git a/server/tests/client.ts b/server/tests/client.ts
index bcbac86e9..b33a653b1 100644
--- a/server/tests/client.ts
+++ b/server/tests/client.ts
@@ -3,17 +3,21 @@
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import * as request from 'supertest' 5import * as request from 'supertest'
6const expect = chai.expect
7
8import { 6import {
9 ServerInfo,
10 flushTests, 7 flushTests,
8 getCustomConfig,
9 getVideosList,
10 killallServers,
11 makeHTMLRequest,
11 runServer, 12 runServer,
13 ServerInfo,
12 serverLogin, 14 serverLogin,
13 uploadVideo, 15 updateCustomConfig,
14 getVideosList, updateCustomConfig, getCustomConfig, killallServers, makeHTMLRequest 16 updateCustomSubConfig,
17 uploadVideo
15} from './utils' 18} from './utils'
16import { CustomConfig } from '../../shared/models/server/custom-config.model' 19
20const expect = chai.expect
17 21
18function checkIndexTags (html: string, title: string, description: string, css: string) { 22function checkIndexTags (html: string, title: string, description: string, css: string) {
19 expect(html).to.contain('<title>' + title + '</title>') 23 expect(html).to.contain('<title>' + title + '</title>')
@@ -117,56 +121,20 @@ describe('Test a client controllers', function () {
117 }) 121 })
118 122
119 it('Should update the customized configuration and have the correct index html tags', async function () { 123 it('Should update the customized configuration and have the correct index html tags', async function () {
120 const newCustomConfig: CustomConfig = { 124 await updateCustomSubConfig(server.url, server.accessToken, {
121 instance: { 125 instance: {
122 name: 'PeerTube updated', 126 name: 'PeerTube updated',
123 shortDescription: 'my short description', 127 shortDescription: 'my short description',
124 description: 'my super description', 128 description: 'my super description',
125 terms: 'my super terms', 129 terms: 'my super terms',
126 defaultClientRoute: '/videos/recently-added', 130 defaultClientRoute: '/videos/recently-added',
127 defaultNSFWPolicy: 'blur' as 'blur', 131 defaultNSFWPolicy: 'blur',
128 customizations: { 132 customizations: {
129 javascript: 'alert("coucou")', 133 javascript: 'alert("coucou")',
130 css: 'body { background-color: red; }' 134 css: 'body { background-color: red; }'
131 } 135 }
132 },
133 services: {
134 twitter: {
135 username: '@Kuja',
136 whitelisted: true
137 }
138 },
139 cache: {
140 previews: {
141 size: 2
142 },
143 captions: {
144 size: 3
145 }
146 },
147 signup: {
148 enabled: false,
149 limit: 5
150 },
151 admin: {
152 email: 'superadmin1@example.com'
153 },
154 user: {
155 videoQuota: 5242881
156 },
157 transcoding: {
158 enabled: true,
159 threads: 1,
160 resolutions: {
161 '240p': false,
162 '360p': true,
163 '480p': true,
164 '720p': false,
165 '1080p': false
166 }
167 } 136 }
168 } 137 })
169 await updateCustomConfig(server.url, server.accessToken, newCustomConfig)
170 138
171 const res = await makeHTMLRequest(server.url, '/videos/trending') 139 const res = await makeHTMLRequest(server.url, '/videos/trending')
172 140
diff --git a/server/tests/fixtures/video_short_mp3_256k.mp4 b/server/tests/fixtures/video_short_mp3_256k.mp4
new file mode 100644
index 000000000..4c1c7b45e
--- /dev/null
+++ b/server/tests/fixtures/video_short_mp3_256k.mp4
Binary files differ
diff --git a/server/tests/fixtures/video_short_no_audio.mp4 b/server/tests/fixtures/video_short_no_audio.mp4
new file mode 100644
index 000000000..329d20fba
--- /dev/null
+++ b/server/tests/fixtures/video_short_no_audio.mp4
Binary files differ
diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts
index 57f95a603..e21614282 100644
--- a/server/tests/utils/server/config.ts
+++ b/server/tests/utils/server/config.ts
@@ -44,6 +44,69 @@ function updateCustomConfig (url: string, token: string, newCustomConfig: Custom
44 }) 44 })
45} 45}
46 46
47function updateCustomSubConfig (url: string, token: string, newConfig: any) {
48 const updateParams: CustomConfig = {
49 instance: {
50 name: 'PeerTube updated',
51 shortDescription: 'my short description',
52 description: 'my super description',
53 terms: 'my super terms',
54 defaultClientRoute: '/videos/recently-added',
55 defaultNSFWPolicy: 'blur',
56 customizations: {
57 javascript: 'alert("coucou")',
58 css: 'body { background-color: red; }'
59 }
60 },
61 services: {
62 twitter: {
63 username: '@MySuperUsername',
64 whitelisted: true
65 }
66 },
67 cache: {
68 previews: {
69 size: 2
70 },
71 captions: {
72 size: 3
73 }
74 },
75 signup: {
76 enabled: false,
77 limit: 5
78 },
79 admin: {
80 email: 'superadmin1@example.com'
81 },
82 user: {
83 videoQuota: 5242881
84 },
85 transcoding: {
86 enabled: true,
87 threads: 1,
88 resolutions: {
89 '240p': false,
90 '360p': true,
91 '480p': true,
92 '720p': false,
93 '1080p': false
94 }
95 },
96 import: {
97 videos: {
98 http: {
99 enabled: false
100 }
101 }
102 }
103 }
104
105 Object.assign(updateParams, newConfig)
106
107 return updateCustomConfig(url, token, updateParams)
108}
109
47function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) { 110function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) {
48 const path = '/api/v1/config/custom' 111 const path = '/api/v1/config/custom'
49 112
@@ -62,5 +125,6 @@ export {
62 getCustomConfig, 125 getCustomConfig,
63 updateCustomConfig, 126 updateCustomConfig,
64 getAbout, 127 getAbout,
65 deleteCustomConfig 128 deleteCustomConfig,
129 updateCustomSubConfig
66} 130}
diff --git a/server/tests/utils/videos/video-imports.ts b/server/tests/utils/videos/video-imports.ts
new file mode 100644
index 000000000..e0f916990
--- /dev/null
+++ b/server/tests/utils/videos/video-imports.ts
@@ -0,0 +1,37 @@
1import { VideoImportCreate } from '../../../../shared/models/videos'
2import { makeGetRequest, makePostBodyRequest } from '..'
3
4function getYoutubeVideoUrl () {
5 return 'https://youtu.be/msX3jv1XdvM'
6}
7
8function importVideo (url: string, token: string, attributes: VideoImportCreate) {
9 const path = '/api/v1/videos/imports'
10
11 return makePostBodyRequest({
12 url,
13 path,
14 token,
15 fields: attributes,
16 statusCodeExpected: 200
17 })
18}
19
20function getMyVideoImports (url: string, token: string) {
21 const path = '/api/v1/users/me/videos/imports'
22
23 return makeGetRequest({
24 url,
25 path,
26 token,
27 statusCodeExpected: 200
28 })
29}
30
31// ---------------------------------------------------------------------------
32
33export {
34 getYoutubeVideoUrl,
35 importVideo,
36 getMyVideoImports
37}
diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts
index 8c49eb02b..b280cccda 100644
--- a/server/tests/utils/videos/videos.ts
+++ b/server/tests/utils/videos/videos.ts
@@ -522,7 +522,9 @@ async function completeVideoCheck (
522 522
523 const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) 523 const minSize = attributeFile.size - ((10 * attributeFile.size) / 100)
524 const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) 524 const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100)
525 expect(file.size).to.be.above(minSize).and.below(maxSize) 525 expect(file.size,
526 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')')
527 .to.be.above(minSize).and.below(maxSize)
526 528
527 { 529 {
528 await testImage(url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) 530 await testImage(url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath)