1 import * as express from 'express'
2 import * as Bluebird from 'bluebird'
3 import * as Sequelize from 'sequelize'
5 import { database as db } from '../../../initializers/database'
7 REQUEST_ENDPOINT_ACTIONS,
9 REQUEST_VIDEO_EVENT_TYPES,
10 REQUEST_VIDEO_QADU_TYPES
11 } from '../../../initializers'
15 remoteVideosValidator,
16 remoteQaduVideosValidator,
17 remoteEventsVideosValidator
18 } from '../../../middlewares'
19 import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers'
20 import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib'
21 import { PodInstance, VideoFileInstance } from '../../../models'
24 RemoteVideoCreateData,
25 RemoteVideoUpdateData,
26 RemoteVideoRemoveData,
27 RemoteVideoReportAbuseData,
28 RemoteQaduVideoRequest,
30 RemoteVideoEventRequest,
32 RemoteVideoChannelCreateData,
33 RemoteVideoChannelUpdateData,
34 RemoteVideoChannelRemoveData,
35 RemoteVideoAuthorRemoveData,
36 RemoteVideoAuthorCreateData
37 } from '../../../../shared'
38 import { VideoInstance } from '../../../models/video/video-interface'
40 const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
42 // Functions to call when processing a remote request
43 // FIXME: use RemoteVideoRequestType as id type
44 const functionsHash: { [ id: string ]: (...args) => Promise<any> } = {}
45 functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper
46 functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper
47 functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper
48 functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper
49 functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper
50 functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper
51 functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper
52 functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper
53 functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper
55 const remoteVideosRouter = express.Router()
57 remoteVideosRouter.post('/',
60 remoteVideosValidator,
64 remoteVideosRouter.post('/qadu',
67 remoteQaduVideosValidator,
71 remoteVideosRouter.post('/events',
74 remoteEventsVideosValidator,
78 // ---------------------------------------------------------------------------
84 // ---------------------------------------------------------------------------
86 function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
87 const requests: RemoteVideoRequest[] = req.body.data
88 const fromPod = res.locals.secure.pod
90 // We need to process in the same order to keep consistency
91 Bluebird.each(requests, request => {
92 const data = request.data
94 // Get the function we need to call in order to process the request
95 const fun = functionsHash[request.type]
96 if (fun === undefined) {
97 logger.error('Unknown remote request type %s.', request.type)
101 return fun.call(this, data, fromPod)
103 .catch(err => logger.error('Error managing remote videos.', err))
105 // Don't block the other pod
106 return res.type('json').status(204).end()
109 function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) {
110 const requests: RemoteQaduVideoRequest[] = req.body.data
111 const fromPod = res.locals.secure.pod
113 Bluebird.each(requests, request => {
114 const videoData = request.data
116 return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod)
118 .catch(err => logger.error('Error managing remote videos.', err))
120 return res.type('json').status(204).end()
123 function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) {
124 const requests: RemoteVideoEventRequest[] = req.body.data
125 const fromPod = res.locals.secure.pod
127 Bluebird.each(requests, request => {
128 const eventData = request.data
130 return processVideosEventsRetryWrapper(eventData, fromPod)
132 .catch(err => logger.error('Error managing remote videos.', err))
134 return res.type('json').status(204).end()
137 async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) {
139 arguments: [ eventData, fromPod ],
140 errorMessage: 'Cannot process videos events with many retries.'
143 await retryTransactionWrapper(processVideosEvents, options)
146 async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) {
147 await db.sequelize.transaction(async t => {
148 const sequelizeOptions = { transaction: t }
149 const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t)
154 switch (eventData.eventType) {
155 case REQUEST_VIDEO_EVENT_TYPES.VIEWS:
156 columnToUpdate = 'views'
157 qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS
160 case REQUEST_VIDEO_EVENT_TYPES.LIKES:
161 columnToUpdate = 'likes'
162 qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES
165 case REQUEST_VIDEO_EVENT_TYPES.DISLIKES:
166 columnToUpdate = 'dislikes'
167 qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES
171 throw new Error('Unknown video event type.')
175 query[columnToUpdate] = eventData.count
177 await videoInstance.increment(query, sequelizeOptions)
179 const qadusParams = [
181 videoId: videoInstance.id,
185 await quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
188 logger.info('Remote video event processed for video with uuid %s.', eventData.uuid)
191 async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
193 arguments: [ videoData, fromPod ],
194 errorMessage: 'Cannot update quick and dirty the remote video with many retries.'
197 await retryTransactionWrapper(quickAndDirtyUpdateVideo, options)
200 async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
203 await db.sequelize.transaction(async t => {
204 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t)
205 const sequelizeOptions = { transaction: t }
207 videoUUID = videoInstance.uuid
209 if (videoData.views) {
210 videoInstance.set('views', videoData.views)
213 if (videoData.likes) {
214 videoInstance.set('likes', videoData.likes)
217 if (videoData.dislikes) {
218 videoInstance.set('dislikes', videoData.dislikes)
221 await videoInstance.save(sequelizeOptions)
224 logger.info('Remote video with uuid %s quick and dirty updated', videoUUID)
227 // Handle retries on fail
228 async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) {
230 arguments: [ videoToCreateData, fromPod ],
231 errorMessage: 'Cannot insert the remote video with many retries.'
234 await retryTransactionWrapper(addRemoteVideo, options)
237 async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) {
238 logger.debug('Adding remote video "%s".', videoToCreateData.uuid)
240 await db.sequelize.transaction(async t => {
241 const sequelizeOptions = {
245 const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid)
246 if (videoFromDatabase) throw new Error('UUID already exists.')
248 const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t)
249 if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.')
251 const tags = videoToCreateData.tags
252 const tagInstances = await db.Tag.findOrCreateTags(tags, t)
255 name: videoToCreateData.name,
256 uuid: videoToCreateData.uuid,
257 category: videoToCreateData.category,
258 licence: videoToCreateData.licence,
259 language: videoToCreateData.language,
260 nsfw: videoToCreateData.nsfw,
261 description: videoToCreateData.truncatedDescription,
262 channelId: videoChannel.id,
263 duration: videoToCreateData.duration,
264 createdAt: videoToCreateData.createdAt,
265 // FIXME: updatedAt does not seems to be considered by Sequelize
266 updatedAt: videoToCreateData.updatedAt,
267 views: videoToCreateData.views,
268 likes: videoToCreateData.likes,
269 dislikes: videoToCreateData.dislikes,
273 const video = db.Video.build(videoData)
274 await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData)
275 const videoCreated = await video.save(sequelizeOptions)
278 for (const fileData of videoToCreateData.files) {
279 const videoFileInstance = db.VideoFile.build({
280 extname: fileData.extname,
281 infoHash: fileData.infoHash,
282 resolution: fileData.resolution,
284 videoId: videoCreated.id
287 tasks.push(videoFileInstance.save(sequelizeOptions))
290 await Promise.all(tasks)
292 await videoCreated.setTags(tagInstances, sequelizeOptions)
295 logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)
298 // Handle retries on fail
299 async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) {
301 arguments: [ videoAttributesToUpdate, fromPod ],
302 errorMessage: 'Cannot update the remote video with many retries'
305 await retryTransactionWrapper(updateRemoteVideo, options)
308 async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) {
309 logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid)
310 let videoInstance: VideoInstance
311 let videoFieldsSave: object
314 await db.sequelize.transaction(async t => {
315 const sequelizeOptions = {
319 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t)
320 videoFieldsSave = videoInstance.toJSON()
321 const tags = videoAttributesToUpdate.tags
323 const tagInstances = await db.Tag.findOrCreateTags(tags, t)
325 videoInstance.set('name', videoAttributesToUpdate.name)
326 videoInstance.set('category', videoAttributesToUpdate.category)
327 videoInstance.set('licence', videoAttributesToUpdate.licence)
328 videoInstance.set('language', videoAttributesToUpdate.language)
329 videoInstance.set('nsfw', videoAttributesToUpdate.nsfw)
330 videoInstance.set('description', videoAttributesToUpdate.truncatedDescription)
331 videoInstance.set('duration', videoAttributesToUpdate.duration)
332 videoInstance.set('createdAt', videoAttributesToUpdate.createdAt)
333 videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt)
334 videoInstance.set('views', videoAttributesToUpdate.views)
335 videoInstance.set('likes', videoAttributesToUpdate.likes)
336 videoInstance.set('dislikes', videoAttributesToUpdate.dislikes)
338 await videoInstance.save(sequelizeOptions)
340 // Remove old video files
341 const videoFileDestroyTasks: Bluebird<void>[] = []
342 for (const videoFile of videoInstance.VideoFiles) {
343 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
345 await Promise.all(videoFileDestroyTasks)
347 const videoFileCreateTasks: Bluebird<VideoFileInstance>[] = []
348 for (const fileData of videoAttributesToUpdate.files) {
349 const videoFileInstance = db.VideoFile.build({
350 extname: fileData.extname,
351 infoHash: fileData.infoHash,
352 resolution: fileData.resolution,
354 videoId: videoInstance.id
357 videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions))
360 await Promise.all(videoFileCreateTasks)
362 await videoInstance.setTags(tagInstances, sequelizeOptions)
365 logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid)
367 if (videoInstance !== undefined && videoFieldsSave !== undefined) {
368 resetSequelizeInstance(videoInstance, videoFieldsSave)
371 // This is just a debug because we will retry the insert
372 logger.debug('Cannot update the remote video.', err)
377 async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
379 arguments: [ videoToRemoveData, fromPod ],
380 errorMessage: 'Cannot remove the remote video channel with many retries.'
383 await retryTransactionWrapper(removeRemoteVideo, options)
386 async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
387 logger.debug('Removing remote video "%s".', videoToRemoveData.uuid)
389 await db.sequelize.transaction(async t => {
390 // We need the instance because we have to remove some other stuffs (thumbnail etc)
391 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t)
392 await videoInstance.destroy({ transaction: t })
395 logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid)
398 async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
400 arguments: [ authorToCreateData, fromPod ],
401 errorMessage: 'Cannot insert the remote video author with many retries.'
404 await retryTransactionWrapper(addRemoteVideoAuthor, options)
407 async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
408 logger.debug('Adding remote video author "%s".', authorToCreateData.uuid)
410 await db.sequelize.transaction(async t => {
411 const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t)
412 if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.')
414 const videoAuthorData = {
415 name: authorToCreateData.name,
416 uuid: authorToCreateData.uuid,
417 userId: null, // Not on our pod
421 const author = db.Author.build(videoAuthorData)
422 await author.save({ transaction: t })
425 logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid)
428 async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
430 arguments: [ authorAttributesToRemove, fromPod ],
431 errorMessage: 'Cannot remove the remote video author with many retries.'
434 await retryTransactionWrapper(removeRemoteVideoAuthor, options)
437 async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
438 logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid)
440 await db.sequelize.transaction(async t => {
441 const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t)
442 await videoAuthor.destroy({ transaction: t })
445 logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid)
448 async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
450 arguments: [ videoChannelToCreateData, fromPod ],
451 errorMessage: 'Cannot insert the remote video channel with many retries.'
454 await retryTransactionWrapper(addRemoteVideoChannel, options)
457 async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
458 logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid)
460 await db.sequelize.transaction(async t => {
461 const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid)
462 if (videoChannelInDatabase) {
463 throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.')
466 const authorUUID = videoChannelToCreateData.ownerUUID
467 const podId = fromPod.id
469 const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t)
470 if (!author) throw new Error('Unknown author UUID' + authorUUID + '.')
472 const videoChannelData = {
473 name: videoChannelToCreateData.name,
474 description: videoChannelToCreateData.description,
475 uuid: videoChannelToCreateData.uuid,
476 createdAt: videoChannelToCreateData.createdAt,
477 updatedAt: videoChannelToCreateData.updatedAt,
482 const videoChannel = db.VideoChannel.build(videoChannelData)
483 await videoChannel.save({ transaction: t })
486 logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid)
489 async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
491 arguments: [ videoChannelAttributesToUpdate, fromPod ],
492 errorMessage: 'Cannot update the remote video channel with many retries.'
495 await retryTransactionWrapper(updateRemoteVideoChannel, options)
498 async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
499 logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid)
501 await db.sequelize.transaction(async t => {
502 const sequelizeOptions = { transaction: t }
504 const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t)
505 videoChannelInstance.set('name', videoChannelAttributesToUpdate.name)
506 videoChannelInstance.set('description', videoChannelAttributesToUpdate.description)
507 videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt)
508 videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt)
510 await videoChannelInstance.save(sequelizeOptions)
513 logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid)
516 async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
518 arguments: [ videoChannelAttributesToRemove, fromPod ],
519 errorMessage: 'Cannot remove the remote video channel with many retries.'
522 await retryTransactionWrapper(removeRemoteVideoChannel, options)
525 async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
526 logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid)
528 await db.sequelize.transaction(async t => {
529 const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t)
530 await videoChannel.destroy({ transaction: t })
533 logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid)
536 async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
538 arguments: [ reportData, fromPod ],
539 errorMessage: 'Cannot create remote abuse video with many retries.'
542 await retryTransactionWrapper(reportAbuseRemoteVideo, options)
545 async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
546 logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID)
548 await db.sequelize.transaction(async t => {
549 const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t)
550 const videoAbuseData = {
551 reporterUsername: reportData.reporterUsername,
552 reason: reportData.reportReason,
553 reporterPodId: fromPod.id,
554 videoId: videoInstance.id
557 await db.VideoAbuse.create(videoAbuseData)
561 logger.info('Remote abuse for video uuid %s created', reportData.videoUUID)
564 async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) {
566 const video = await db.Video.loadLocalVideoByUUID(id, t)
568 if (!video) throw new Error('Video ' + id + ' not found')
572 logger.error('Cannot load owned video from id.', { error: err.stack, id })
577 async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) {
579 const video = await db.Video.loadByHostAndUUID(podHost, uuid, t)
580 if (!video) throw new Error('Video not found')
584 logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid })