1 import * as express from 'express'
2 import * as Bluebird from 'bluebird'
3 import * as Sequelize from 'sequelize'
5 import { database as db } from '../../../initializers/database'
7 REQUEST_ENDPOINT_ACTIONS,
9 REQUEST_VIDEO_EVENT_TYPES,
10 REQUEST_VIDEO_QADU_TYPES
11 } from '../../../initializers'
15 remoteVideosValidator,
16 remoteQaduVideosValidator,
17 remoteEventsVideosValidator
18 } from '../../../middlewares'
19 import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers'
20 import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib'
21 import { PodInstance, VideoFileInstance } from '../../../models'
24 RemoteVideoCreateData,
25 RemoteVideoUpdateData,
26 RemoteVideoRemoveData,
27 RemoteVideoReportAbuseData,
28 RemoteQaduVideoRequest,
30 RemoteVideoEventRequest,
32 RemoteVideoChannelCreateData,
33 RemoteVideoChannelUpdateData,
34 RemoteVideoChannelRemoveData,
35 RemoteVideoAuthorRemoveData,
36 RemoteVideoAuthorCreateData
37 } from '../../../../shared'
38 import { VideoInstance } from '../../../models/video/video-interface'
40 const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
42 // Functions to call when processing a remote request
43 // FIXME: use RemoteVideoRequestType as id type
44 const functionsHash: { [ id: string ]: (...args) => Promise<any> } = {}
45 functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper
46 functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper
47 functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper
48 functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper
49 functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper
50 functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper
51 functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper
52 functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper
53 functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper
55 const remoteVideosRouter = express.Router()
57 remoteVideosRouter.post('/',
60 remoteVideosValidator,
64 remoteVideosRouter.post('/qadu',
67 remoteQaduVideosValidator,
71 remoteVideosRouter.post('/events',
74 remoteEventsVideosValidator,
78 // ---------------------------------------------------------------------------
84 // ---------------------------------------------------------------------------
86 function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) {
87 const requests: RemoteVideoRequest[] = req.body.data
88 const fromPod = res.locals.secure.pod
90 // We need to process in the same order to keep consistency
91 Bluebird.each(requests, request => {
92 const data = request.data
94 // Get the function we need to call in order to process the request
95 const fun = functionsHash[request.type]
96 if (fun === undefined) {
97 logger.error('Unknown remote request type %s.', request.type)
101 return fun.call(this, data, fromPod)
103 .catch(err => logger.error('Error managing remote videos.', err))
105 // Don't block the other pod
106 return res.type('json').status(204).end()
109 function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) {
110 const requests: RemoteQaduVideoRequest[] = req.body.data
111 const fromPod = res.locals.secure.pod
113 Bluebird.each(requests, request => {
114 const videoData = request.data
116 return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod)
118 .catch(err => logger.error('Error managing remote videos.', err))
120 return res.type('json').status(204).end()
123 function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) {
124 const requests: RemoteVideoEventRequest[] = req.body.data
125 const fromPod = res.locals.secure.pod
127 Bluebird.each(requests, request => {
128 const eventData = request.data
130 return processVideosEventsRetryWrapper(eventData, fromPod)
132 .catch(err => logger.error('Error managing remote videos.', err))
134 return res.type('json').status(204).end()
137 async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) {
139 arguments: [ eventData, fromPod ],
140 errorMessage: 'Cannot process videos events with many retries.'
143 await retryTransactionWrapper(processVideosEvents, options)
146 async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) {
147 await db.sequelize.transaction(async t => {
148 const sequelizeOptions = { transaction: t }
149 const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t)
154 switch (eventData.eventType) {
155 case REQUEST_VIDEO_EVENT_TYPES.VIEWS:
156 columnToUpdate = 'views'
157 qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS
160 case REQUEST_VIDEO_EVENT_TYPES.LIKES:
161 columnToUpdate = 'likes'
162 qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES
165 case REQUEST_VIDEO_EVENT_TYPES.DISLIKES:
166 columnToUpdate = 'dislikes'
167 qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES
171 throw new Error('Unknown video event type.')
175 query[columnToUpdate] = eventData.count
177 await videoInstance.increment(query, sequelizeOptions)
179 const qadusParams = [
181 videoId: videoInstance.id,
185 await quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
188 logger.info('Remote video event processed for video with uuid %s.', eventData.uuid)
191 async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
193 arguments: [ videoData, fromPod ],
194 errorMessage: 'Cannot update quick and dirty the remote video with many retries.'
197 await retryTransactionWrapper(quickAndDirtyUpdateVideo, options)
200 async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
203 await db.sequelize.transaction(async t => {
204 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t)
205 const sequelizeOptions = { transaction: t }
207 videoUUID = videoInstance.uuid
209 if (videoData.views) {
210 videoInstance.set('views', videoData.views)
213 if (videoData.likes) {
214 videoInstance.set('likes', videoData.likes)
217 if (videoData.dislikes) {
218 videoInstance.set('dislikes', videoData.dislikes)
221 await videoInstance.save(sequelizeOptions)
224 logger.info('Remote video with uuid %s quick and dirty updated', videoUUID)
227 // Handle retries on fail
228 async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) {
230 arguments: [ videoToCreateData, fromPod ],
231 errorMessage: 'Cannot insert the remote video with many retries.'
234 await retryTransactionWrapper(addRemoteVideo, options)
237 async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) {
238 logger.debug('Adding remote video "%s".', videoToCreateData.uuid)
240 await db.sequelize.transaction(async t => {
241 const sequelizeOptions = {
245 const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid)
246 if (videoFromDatabase) throw new Error('UUID already exists.')
248 const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t)
249 if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.')
251 const tags = videoToCreateData.tags
252 const tagInstances = await db.Tag.findOrCreateTags(tags, t)
255 name: videoToCreateData.name,
256 uuid: videoToCreateData.uuid,
257 category: videoToCreateData.category,
258 licence: videoToCreateData.licence,
259 language: videoToCreateData.language,
260 nsfw: videoToCreateData.nsfw,
261 description: videoToCreateData.truncatedDescription,
262 channelId: videoChannel.id,
263 duration: videoToCreateData.duration,
264 createdAt: videoToCreateData.createdAt,
265 // FIXME: updatedAt does not seems to be considered by Sequelize
266 updatedAt: videoToCreateData.updatedAt,
267 views: videoToCreateData.views,
268 likes: videoToCreateData.likes,
269 dislikes: videoToCreateData.dislikes,
271 privacy: videoToCreateData.privacy
274 const video = db.Video.build(videoData)
275 await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData)
276 const videoCreated = await video.save(sequelizeOptions)
279 for (const fileData of videoToCreateData.files) {
280 const videoFileInstance = db.VideoFile.build({
281 extname: fileData.extname,
282 infoHash: fileData.infoHash,
283 resolution: fileData.resolution,
285 videoId: videoCreated.id
288 tasks.push(videoFileInstance.save(sequelizeOptions))
291 await Promise.all(tasks)
293 await videoCreated.setTags(tagInstances, sequelizeOptions)
296 logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)
299 // Handle retries on fail
300 async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) {
302 arguments: [ videoAttributesToUpdate, fromPod ],
303 errorMessage: 'Cannot update the remote video with many retries'
306 await retryTransactionWrapper(updateRemoteVideo, options)
309 async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) {
310 logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid)
311 let videoInstance: VideoInstance
312 let videoFieldsSave: object
315 await db.sequelize.transaction(async t => {
316 const sequelizeOptions = {
320 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t)
321 videoFieldsSave = videoInstance.toJSON()
322 const tags = videoAttributesToUpdate.tags
324 const tagInstances = await db.Tag.findOrCreateTags(tags, t)
326 videoInstance.set('name', videoAttributesToUpdate.name)
327 videoInstance.set('category', videoAttributesToUpdate.category)
328 videoInstance.set('licence', videoAttributesToUpdate.licence)
329 videoInstance.set('language', videoAttributesToUpdate.language)
330 videoInstance.set('nsfw', videoAttributesToUpdate.nsfw)
331 videoInstance.set('description', videoAttributesToUpdate.truncatedDescription)
332 videoInstance.set('duration', videoAttributesToUpdate.duration)
333 videoInstance.set('createdAt', videoAttributesToUpdate.createdAt)
334 videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt)
335 videoInstance.set('views', videoAttributesToUpdate.views)
336 videoInstance.set('likes', videoAttributesToUpdate.likes)
337 videoInstance.set('dislikes', videoAttributesToUpdate.dislikes)
338 videoInstance.set('privacy', videoAttributesToUpdate.privacy)
340 await videoInstance.save(sequelizeOptions)
342 // Remove old video files
343 const videoFileDestroyTasks: Bluebird<void>[] = []
344 for (const videoFile of videoInstance.VideoFiles) {
345 videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions))
347 await Promise.all(videoFileDestroyTasks)
349 const videoFileCreateTasks: Bluebird<VideoFileInstance>[] = []
350 for (const fileData of videoAttributesToUpdate.files) {
351 const videoFileInstance = db.VideoFile.build({
352 extname: fileData.extname,
353 infoHash: fileData.infoHash,
354 resolution: fileData.resolution,
356 videoId: videoInstance.id
359 videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions))
362 await Promise.all(videoFileCreateTasks)
364 await videoInstance.setTags(tagInstances, sequelizeOptions)
367 logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid)
369 if (videoInstance !== undefined && videoFieldsSave !== undefined) {
370 resetSequelizeInstance(videoInstance, videoFieldsSave)
373 // This is just a debug because we will retry the insert
374 logger.debug('Cannot update the remote video.', err)
379 async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
381 arguments: [ videoToRemoveData, fromPod ],
382 errorMessage: 'Cannot remove the remote video channel with many retries.'
385 await retryTransactionWrapper(removeRemoteVideo, options)
388 async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) {
389 logger.debug('Removing remote video "%s".', videoToRemoveData.uuid)
391 await db.sequelize.transaction(async t => {
392 // We need the instance because we have to remove some other stuffs (thumbnail etc)
393 const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t)
394 await videoInstance.destroy({ transaction: t })
397 logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid)
400 async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
402 arguments: [ authorToCreateData, fromPod ],
403 errorMessage: 'Cannot insert the remote video author with many retries.'
406 await retryTransactionWrapper(addRemoteVideoAuthor, options)
409 async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) {
410 logger.debug('Adding remote video author "%s".', authorToCreateData.uuid)
412 await db.sequelize.transaction(async t => {
413 const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t)
414 if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.')
416 const videoAuthorData = {
417 name: authorToCreateData.name,
418 uuid: authorToCreateData.uuid,
419 userId: null, // Not on our pod
423 const author = db.Author.build(videoAuthorData)
424 await author.save({ transaction: t })
427 logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid)
430 async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
432 arguments: [ authorAttributesToRemove, fromPod ],
433 errorMessage: 'Cannot remove the remote video author with many retries.'
436 await retryTransactionWrapper(removeRemoteVideoAuthor, options)
439 async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) {
440 logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid)
442 await db.sequelize.transaction(async t => {
443 const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t)
444 await videoAuthor.destroy({ transaction: t })
447 logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid)
450 async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
452 arguments: [ videoChannelToCreateData, fromPod ],
453 errorMessage: 'Cannot insert the remote video channel with many retries.'
456 await retryTransactionWrapper(addRemoteVideoChannel, options)
459 async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) {
460 logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid)
462 await db.sequelize.transaction(async t => {
463 const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid)
464 if (videoChannelInDatabase) {
465 throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.')
468 const authorUUID = videoChannelToCreateData.ownerUUID
469 const podId = fromPod.id
471 const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t)
472 if (!author) throw new Error('Unknown author UUID' + authorUUID + '.')
474 const videoChannelData = {
475 name: videoChannelToCreateData.name,
476 description: videoChannelToCreateData.description,
477 uuid: videoChannelToCreateData.uuid,
478 createdAt: videoChannelToCreateData.createdAt,
479 updatedAt: videoChannelToCreateData.updatedAt,
484 const videoChannel = db.VideoChannel.build(videoChannelData)
485 await videoChannel.save({ transaction: t })
488 logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid)
491 async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
493 arguments: [ videoChannelAttributesToUpdate, fromPod ],
494 errorMessage: 'Cannot update the remote video channel with many retries.'
497 await retryTransactionWrapper(updateRemoteVideoChannel, options)
500 async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) {
501 logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid)
503 await db.sequelize.transaction(async t => {
504 const sequelizeOptions = { transaction: t }
506 const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t)
507 videoChannelInstance.set('name', videoChannelAttributesToUpdate.name)
508 videoChannelInstance.set('description', videoChannelAttributesToUpdate.description)
509 videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt)
510 videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt)
512 await videoChannelInstance.save(sequelizeOptions)
515 logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid)
518 async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
520 arguments: [ videoChannelAttributesToRemove, fromPod ],
521 errorMessage: 'Cannot remove the remote video channel with many retries.'
524 await retryTransactionWrapper(removeRemoteVideoChannel, options)
527 async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) {
528 logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid)
530 await db.sequelize.transaction(async t => {
531 const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t)
532 await videoChannel.destroy({ transaction: t })
535 logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid)
538 async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
540 arguments: [ reportData, fromPod ],
541 errorMessage: 'Cannot create remote abuse video with many retries.'
544 await retryTransactionWrapper(reportAbuseRemoteVideo, options)
547 async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) {
548 logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID)
550 await db.sequelize.transaction(async t => {
551 const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t)
552 const videoAbuseData = {
553 reporterUsername: reportData.reporterUsername,
554 reason: reportData.reportReason,
555 reporterPodId: fromPod.id,
556 videoId: videoInstance.id
559 await db.VideoAbuse.create(videoAbuseData)
563 logger.info('Remote abuse for video uuid %s created', reportData.videoUUID)
566 async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) {
568 const video = await db.Video.loadLocalVideoByUUID(id, t)
570 if (!video) throw new Error('Video ' + id + ' not found')
574 logger.error('Cannot load owned video from id.', { error: err.stack, id })
579 async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) {
581 const video = await db.Video.loadByHostAndUUID(podHost, uuid, t)
582 if (!video) throw new Error('Video not found')
586 logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid })