1 import * as express from 'express'
2 import * as Bluebird from 'bluebird'
3 import * as Sequelize from 'sequelize'
5 import { database as db } from '../../../initializers/database'
7 REQUEST_ENDPOINT_ACTIONS,
9 REQUEST_VIDEO_EVENT_TYPES,
10 REQUEST_VIDEO_QADU_TYPES
11 } from '../../../initializers'
15 remoteVideosValidator,
16 remoteQaduVideosValidator,
17 remoteEventsVideosValidator
18 } from '../../../middlewares'
19 import { logger, retryTransactionWrapper } from '../../../helpers'
20 import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib'
21 import { PodInstance, VideoFileInstance } from '../../../models'
24 RemoteVideoCreateData,
25 RemoteVideoUpdateData,
26 RemoteVideoRemoveData,
27 RemoteVideoReportAbuseData,
28 RemoteQaduVideoRequest,
30 RemoteVideoEventRequest,
32 RemoteVideoChannelCreateData,
33 RemoteVideoChannelUpdateData,
34 RemoteVideoChannelRemoveData,
35 RemoteVideoAuthorRemoveData,
36 RemoteVideoAuthorCreateData
37 } from '../../../../shared'
39 const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
41 // Functions to call when processing a remote request
42 // FIXME: use RemoteVideoRequestType as id type
43 const functionsHash: { [ id: string ]: (...args) => Promise<any> } = {}
44 functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper
45 functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper
46 functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper
47 functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper
48 functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper
49 functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper
50 functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper
51 functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper
52 functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper
54 const remoteVideosRouter = express.Router()
56 remoteVideosRouter.post('/',
59 remoteVideosValidator,
63 remoteVideosRouter.post('/qadu',
66 remoteQaduVideosValidator,
70 remoteVideosRouter.post('/events',
73 remoteEventsVideosValidator,
77 // ---------------------------------------------------------------------------
83 // ---------------------------------------------------------------------------
85 function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
86 const requests: RemoteVideoRequest[] = req.body.data
87 const fromPod = res.locals.secure.pod
89 // We need to process in the same order to keep consistency
90 Bluebird.each(requests, request => {
91 const data = request.data
93 // Get the function we need to call in order to process the request
94 const fun = functionsHash[request.type]
95 if (fun === undefined) {
96 logger.error('Unknown remote request type %s.', request.type)
100 return fun.call(this, data, fromPod)
102 .catch(err => logger.error('Error managing remote videos.', err))
104 // Don't block the other pod
105 return res.type('json').status(204).end()
108 function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) {
109 const requests: RemoteQaduVideoRequest[] = req.body.data
110 const fromPod = res.locals.secure.pod
112 Bluebird.each(requests, request => {
113 const videoData = request.data
115 return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod)
117 .catch(err => logger.error('Error managing remote videos.', err))
119 return res.type('json').status(204).end()
122 function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) {
123 const requests: RemoteVideoEventRequest[] = req.body.data
124 const fromPod = res.locals.secure.pod
126 Bluebird.each(requests, request => {
127 const eventData = request.data
129 return processVideosEventsRetryWrapper(eventData, fromPod)
131 .catch(err => logger.error('Error managing remote videos.', err))
133 return res.type('json').status(204).end()
136 async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) {
138 arguments: [ eventData, fromPod ],
139 errorMessage: 'Cannot process videos events with many retries.'
142 await retryTransactionWrapper(processVideosEvents, options)
145 async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) {
146 await db.sequelize.transaction(async t => {
147 const sequelizeOptions = { transaction: t }
148 const videoInstance = await fetchVideoByUUID(eventData.uuid, t)
153 switch (eventData.eventType) {
154 case REQUEST_VIDEO_EVENT_TYPES.VIEWS:
155 columnToUpdate = 'views'
156 qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS
159 case REQUEST_VIDEO_EVENT_TYPES.LIKES:
160 columnToUpdate = 'likes'
161 qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES
164 case REQUEST_VIDEO_EVENT_TYPES.DISLIKES:
165 columnToUpdate = 'dislikes'
166 qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES
170 throw new Error('Unknown video event type.')
174 query[columnToUpdate] = eventData.count
176 await videoInstance.increment(query, sequelizeOptions)
178 const qadusParams = [
180 videoId: videoInstance.id,
184 await quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
187 logger.info('Remote video event processed for video with uuid %s.', eventData.uuid)
190 async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
192 arguments: [ videoData, fromPod ],
193 errorMessage: 'Cannot update quick and dirty the remote video with many retries.'
196 await retryTransactionWrapper(quickAndDirtyUpdateVideo, options)
199 async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
202 await db.sequelize.transaction(async t => {
203 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t)
204 const sequelizeOptions = { transaction: t }
206 videoUUID = videoInstance.uuid
208 if (videoData.views) {
209 videoInstance.set('views', videoData.views)
212 if (videoData.likes) {
213 videoInstance.set('likes', videoData.likes)
216 if (videoData.dislikes) {
217 videoInstance.set('dislikes', videoData.dislikes)
220 await videoInstance.save(sequelizeOptions)
223 logger.info('Remote video with uuid %s quick and dirty updated', videoUUID)
226 // Handle retries on fail
227 async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) {
229 arguments: [ videoToCreateData, fromPod ],
230 errorMessage: 'Cannot insert the remote video with many retries.'
233 await retryTransactionWrapper(addRemoteVideo, options)
236 async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) {
237 logger.debug('Adding remote video "%s".', videoToCreateData.uuid)
239 await db.sequelize.transaction(async t => {
240 const sequelizeOptions = {
244 const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid)
245 if (videoFromDatabase) throw new Error('UUID already exists.')
247 const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t)
248 if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.')
250 const tags = videoToCreateData.tags
251 const tagInstances = await db.Tag.findOrCreateTags(tags, t)
254 name: videoToCreateData.name,
255 uuid: videoToCreateData.uuid,
256 category: videoToCreateData.category,
257 licence: videoToCreateData.licence,
258 language: videoToCreateData.language,
259 nsfw: videoToCreateData.nsfw,
260 description: videoToCreateData.description,
261 channelId: videoChannel.id,
262 duration: videoToCreateData.duration,
263 createdAt: videoToCreateData.createdAt,
264 // FIXME: updatedAt does not seems to be considered by Sequelize
265 updatedAt: videoToCreateData.updatedAt,
266 views: videoToCreateData.views,
267 likes: videoToCreateData.likes,
268 dislikes: videoToCreateData.dislikes,
272 const video = db.Video.build(videoData)
273 await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData)
274 const videoCreated = await video.save(sequelizeOptions)
277 for (const fileData of videoToCreateData.files) {
278 const videoFileInstance = db.VideoFile.build({
279 extname: fileData.extname,
280 infoHash: fileData.infoHash,
281 resolution: fileData.resolution,
283 videoId: videoCreated.id
286 tasks.push(videoFileInstance.save(sequelizeOptions))
289 await Promise.all(tasks)
291 await videoCreated.setTags(tagInstances, sequelizeOptions)
294 logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)
297 // Handle retries on fail
298 async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) {
300 arguments: [ videoAttributesToUpdate, fromPod ],
301 errorMessage: 'Cannot update the remote video with many retries'
304 await retryTransactionWrapper(updateRemoteVideo, options)
307 async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) {
308 logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid)
311 await db.sequelize.transaction(async t => {
312 const sequelizeOptions = {
316 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t)
317 const tags = videoAttributesToUpdate.tags
319 const tagInstances = await db.Tag.findOrCreateTags(tags, t)
321 videoInstance.set('name', videoAttributesToUpdate.name)
322 videoInstance.set('category', videoAttributesToUpdate.category)
323 videoInstance.set('licence', videoAttributesToUpdate.licence)
324 videoInstance.set('language', videoAttributesToUpdate.language)
325 videoInstance.set('nsfw', videoAttributesToUpdate.nsfw)
326 videoInstance.set('description', videoAttributesToUpdate.description)
327 videoInstance.set('duration', videoAttributesToUpdate.duration)
328 videoInstance.set('createdAt', videoAttributesToUpdate.createdAt)
329 videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt)
330 videoInstance.set('views', videoAttributesToUpdate.views)
331 videoInstance.set('likes', videoAttributesToUpdate.likes)
332 videoInstance.set('dislikes', videoAttributesToUpdate.dislikes)
334 await videoInstance.save(sequelizeOptions)
336 // Remove old video files
337 const videoFileDestroyTasks: Bluebird<void>[] = []
338 for (const videoFile of videoInstance.VideoFiles) {
339 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
341 await Promise.all(videoFileDestroyTasks)
343 const videoFileCreateTasks: Bluebird<VideoFileInstance>[] = []
344 for (const fileData of videoAttributesToUpdate.files) {
345 const videoFileInstance = db.VideoFile.build({
346 extname: fileData.extname,
347 infoHash: fileData.infoHash,
348 resolution: fileData.resolution,
350 videoId: videoInstance.id
353 videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions))
356 await Promise.all(videoFileCreateTasks)
358 await videoInstance.setTags(tagInstances, sequelizeOptions)
361 logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid)
363 // This is just a debug because we will retry the insert
364 logger.debug('Cannot update the remote video.', err)
369 async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
371 arguments: [ videoToRemoveData, fromPod ],
372 errorMessage: 'Cannot remove the remote video channel with many retries.'
375 await retryTransactionWrapper(removeRemoteVideo, options)
378 async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
379 logger.debug('Removing remote video "%s".', videoToRemoveData.uuid)
381 await db.sequelize.transaction(async t => {
382 // We need the instance because we have to remove some other stuffs (thumbnail etc)
383 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t)
384 await videoInstance.destroy({ transaction: t })
387 logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid)
390 async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
392 arguments: [ authorToCreateData, fromPod ],
393 errorMessage: 'Cannot insert the remote video author with many retries.'
396 await retryTransactionWrapper(addRemoteVideoAuthor, options)
399 async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
400 logger.debug('Adding remote video author "%s".', authorToCreateData.uuid)
402 await db.sequelize.transaction(async t => {
403 const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t)
404 if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.')
406 const videoAuthorData = {
407 name: authorToCreateData.name,
408 uuid: authorToCreateData.uuid,
409 userId: null, // Not on our pod
413 const author = db.Author.build(videoAuthorData)
414 await author.save({ transaction: t })
417 logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid)
420 async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
422 arguments: [ authorAttributesToRemove, fromPod ],
423 errorMessage: 'Cannot remove the remote video author with many retries.'
426 await retryTransactionWrapper(removeRemoteVideoAuthor, options)
429 async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
430 logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid)
432 await db.sequelize.transaction(async t => {
433 const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t)
434 await videoAuthor.destroy({ transaction: t })
437 logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid)
440 async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
442 arguments: [ videoChannelToCreateData, fromPod ],
443 errorMessage: 'Cannot insert the remote video channel with many retries.'
446 await retryTransactionWrapper(addRemoteVideoChannel, options)
449 async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
450 logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid)
452 await db.sequelize.transaction(async t => {
453 const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid)
454 if (videoChannelInDatabase) {
455 throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.')
458 const authorUUID = videoChannelToCreateData.ownerUUID
459 const podId = fromPod.id
461 const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t)
462 if (!author) throw new Error('Unknown author UUID' + authorUUID + '.')
464 const videoChannelData = {
465 name: videoChannelToCreateData.name,
466 description: videoChannelToCreateData.description,
467 uuid: videoChannelToCreateData.uuid,
468 createdAt: videoChannelToCreateData.createdAt,
469 updatedAt: videoChannelToCreateData.updatedAt,
474 const videoChannel = db.VideoChannel.build(videoChannelData)
475 await videoChannel.save({ transaction: t })
478 logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid)
481 async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
483 arguments: [ videoChannelAttributesToUpdate, fromPod ],
484 errorMessage: 'Cannot update the remote video channel with many retries.'
487 await retryTransactionWrapper(updateRemoteVideoChannel, options)
490 async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
491 logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid)
493 await db.sequelize.transaction(async t => {
494 const sequelizeOptions = { transaction: t }
496 const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t)
497 videoChannelInstance.set('name', videoChannelAttributesToUpdate.name)
498 videoChannelInstance.set('description', videoChannelAttributesToUpdate.description)
499 videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt)
500 videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt)
502 await videoChannelInstance.save(sequelizeOptions)
505 logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid)
508 async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
510 arguments: [ videoChannelAttributesToRemove, fromPod ],
511 errorMessage: 'Cannot remove the remote video channel with many retries.'
514 await retryTransactionWrapper(removeRemoteVideoChannel, options)
517 async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
518 logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid)
520 await db.sequelize.transaction(async t => {
521 const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t)
522 await videoChannel.destroy({ transaction: t })
525 logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid)
528 async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
530 arguments: [ reportData, fromPod ],
531 errorMessage: 'Cannot create remote abuse video with many retries.'
534 await retryTransactionWrapper(reportAbuseRemoteVideo, options)
537 async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
538 logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID)
540 await db.sequelize.transaction(async t => {
541 const videoInstance = await fetchVideoByUUID(reportData.videoUUID, t)
542 const videoAbuseData = {
543 reporterUsername: reportData.reporterUsername,
544 reason: reportData.reportReason,
545 reporterPodId: fromPod.id,
546 videoId: videoInstance.id
549 await db.VideoAbuse.create(videoAbuseData)
553 logger.info('Remote abuse for video uuid %s created', reportData.videoUUID)
556 async function fetchVideoByUUID (id: string, t: Sequelize.Transaction) {
558 const video = await db.Video.loadByUUID(id, t)
560 if (!video) throw new Error('Video ' + id + ' not found')
564 logger.error('Cannot load owned video from id.', { error: err.stack, id })
569 async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) {
571 const video = await db.Video.loadByHostAndUUID(podHost, uuid, t)
572 if (!video) throw new Error('Video not found')
576 logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid })