From 0d0e8dd0904b380b70e19ebcb4763d65601c4632 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Nov 2017 14:34:45 +0100 Subject: [PATCH] Continue activitypub --- server/controllers/activitypub/inbox.ts | 30 +- server/controllers/activitypub/pods.ts | 138 +-- server/controllers/activitypub/videos.ts | 928 +++++++----------- server/controllers/api/videos/index.ts | 30 +- server/helpers/activitypub.ts | 45 +- .../custom-validators/activitypub/activity.ts | 34 + .../custom-validators/activitypub/index.ts | 1 + .../custom-validators/activitypub/misc.ts | 12 +- .../custom-validators/activitypub/videos.ts | 223 ++--- server/helpers/custom-validators/index.ts | 1 - .../custom-validators/video-authors.ts | 45 - server/helpers/custom-validators/videos.ts | 16 +- server/helpers/requests.ts | 11 + server/initializers/constants.ts | 56 +- server/lib/activitypub/misc.ts | 77 ++ server/lib/activitypub/process-add.ts | 72 ++ server/lib/activitypub/process-create.ts | 104 +- server/lib/activitypub/process-update.ts | 127 ++- .../validators/activitypub/activity.ts | 21 + .../validators/activitypub/videos.ts | 61 -- .../models/video/video-channel-interface.ts | 6 +- server/models/video/video-channel.ts | 35 +- server/models/video/video-interface.ts | 11 +- server/models/video/video.ts | 24 +- shared/models/activitypub/activity.ts | 9 +- .../objects/video-channel-object.ts | 5 +- .../objects/video-torrent-object.ts | 1 + 27 files changed, 1038 insertions(+), 1085 deletions(-) create mode 100644 server/helpers/custom-validators/activitypub/activity.ts delete mode 100644 server/helpers/custom-validators/video-authors.ts create mode 100644 server/lib/activitypub/misc.ts create mode 100644 server/lib/activitypub/process-add.ts create mode 100644 server/middlewares/validators/activitypub/activity.ts delete mode 100644 server/middlewares/validators/activitypub/videos.ts diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts index 79d989c2c..eee217650 100644 --- a/server/controllers/activitypub/inbox.ts +++ b/server/controllers/activitypub/inbox.ts @@ -1,26 +1,15 @@ import * as express from 'express' - -import { - processCreateActivity, - processUpdateActivity, - processFlagActivity -} from '../../lib' -import { - Activity, - ActivityType, - RootActivity, - ActivityPubCollection, - ActivityPubOrderedCollection -} from '../../../shared' -import { - signatureValidator, - checkSignature, - asyncMiddleware -} from '../../middlewares' +import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, ActivityType, RootActivity } from '../../../shared' import { logger } from '../../helpers' +import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity' +import { processCreateActivity, processFlagActivity, processUpdateActivity } from '../../lib' +import { processAddActivity } from '../../lib/activitypub/process-add' +import { asyncMiddleware, checkSignature, signatureValidator } from '../../middlewares' +import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' const processActivity: { [ P in ActivityType ]: (activity: Activity) => Promise } = { Create: processCreateActivity, + Add: processAddActivity, Update: processUpdateActivity, Flag: processFlagActivity } @@ -30,7 +19,7 @@ const inboxRouter = express.Router() inboxRouter.post('/', signatureValidator, asyncMiddleware(checkSignature), - // inboxValidator, + activityPubValidator, asyncMiddleware(inboxController) ) @@ -54,6 +43,9 @@ async function inboxController (req: express.Request, res: express.Response, nex activities = [ rootActivity as Activity ] } + // Only keep activities we are able to process + activities = activities.filter(a => isActivityValid(a)) + await processActivities(activities) res.status(204).end() diff --git a/server/controllers/activitypub/pods.ts b/server/controllers/activitypub/pods.ts index 326eb61ac..6cce57c1c 100644 --- a/server/controllers/activitypub/pods.ts +++ b/server/controllers/activitypub/pods.ts @@ -1,69 +1,69 @@ -import * as express from 'express' - -import { database as db } from '../../../initializers/database' -import { - checkSignature, - signatureValidator, - setBodyHostPort, - remotePodsAddValidator, - asyncMiddleware -} from '../../../middlewares' -import { sendOwnedDataToPod } from '../../../lib' -import { getMyPublicCert, getFormattedObjects } from '../../../helpers' -import { CONFIG } from '../../../initializers' -import { PodInstance } from '../../../models' -import { PodSignature, Pod as FormattedPod } from '../../../../shared' - -const remotePodsRouter = express.Router() - -remotePodsRouter.post('/remove', - signatureValidator, - checkSignature, - asyncMiddleware(removePods) -) - -remotePodsRouter.post('/list', - asyncMiddleware(remotePodsList) -) - -remotePodsRouter.post('/add', - setBodyHostPort, // We need to modify the host before running the validator! - remotePodsAddValidator, - asyncMiddleware(addPods) -) - -// --------------------------------------------------------------------------- - -export { - remotePodsRouter -} - -// --------------------------------------------------------------------------- - -async function addPods (req: express.Request, res: express.Response, next: express.NextFunction) { - const information = req.body - - const pod = db.Pod.build(information) - const podCreated = await pod.save() - - await sendOwnedDataToPod(podCreated.id) - - const cert = await getMyPublicCert() - return res.json({ cert, email: CONFIG.ADMIN.EMAIL }) -} - -async function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) { - const pods = await db.Pod.list() - - return res.json(getFormattedObjects(pods, pods.length)) -} - -async function removePods (req: express.Request, res: express.Response, next: express.NextFunction) { - const signature: PodSignature = req.body.signature - const host = signature.host - - const pod = await db.Pod.loadByHost(host) - await pod.destroy() - - return res.type('json').status(204).end() -} +// import * as express from 'express' +// +// import { database as db } from '../../../initializers/database' +// import { +// checkSignature, +// signatureValidator, +// setBodyHostPort, +// remotePodsAddValidator, +// asyncMiddleware +// } from '../../../middlewares' +// import { sendOwnedDataToPod } from '../../../lib' +// import { getMyPublicCert, getFormattedObjects } from '../../../helpers' +// import { CONFIG } from '../../../initializers' +// import { PodInstance } from '../../../models' +// import { PodSignature, Pod as FormattedPod } from '../../../../shared' +// +// const remotePodsRouter = express.Router() +// +// remotePodsRouter.post('/remove', +// signatureValidator, +// checkSignature, +// asyncMiddleware(removePods) +// ) +// +// remotePodsRouter.post('/list', +// asyncMiddleware(remotePodsList) +// ) +// +// remotePodsRouter.post('/add', +// setBodyHostPort, // We need to modify the host before running the validator! +// remotePodsAddValidator, +// asyncMiddleware(addPods) +// ) +// +// // --------------------------------------------------------------------------- +// +// export { +// remotePodsRouter +// } +// +// // --------------------------------------------------------------------------- +// +// async function addPods (req: express.Request, res: express.Response, next: express.NextFunction) { +// const information = req.body +// +// const pod = db.Pod.build(information) +// const podCreated = await pod.save() +// +// await sendOwnedDataToPod(podCreated.id) +// +// const cert = await getMyPublicCert() +// return res.json({ cert, email: CONFIG.ADMIN.EMAIL }) +// } +// +// async function remotePodsList (req: express.Request, res: express.Response, next: express.NextFunction) { +// const pods = await db.Pod.list() +// +// return res.json(getFormattedObjects(pods, pods.length)) +// } +// +// async function removePods (req: express.Request, res: express.Response, next: express.NextFunction) { +// const signature: PodSignature = req.body.signature +// const host = signature.host +// +// const pod = await db.Pod.loadByHost(host) +// await pod.destroy() +// +// return res.type('json').status(204).end() +// } diff --git a/server/controllers/activitypub/videos.ts b/server/controllers/activitypub/videos.ts index cba47f0a1..9a1868ff7 100644 --- a/server/controllers/activitypub/videos.ts +++ b/server/controllers/activitypub/videos.ts @@ -1,589 +1,339 @@ -import * as express from 'express' -import * as Bluebird from 'bluebird' -import * as Sequelize from 'sequelize' - -import { database as db } from '../../../initializers/database' -import { - REQUEST_ENDPOINT_ACTIONS, - REQUEST_ENDPOINTS, - REQUEST_VIDEO_EVENT_TYPES, - REQUEST_VIDEO_QADU_TYPES -} from '../../../initializers' -import { - checkSignature, - signatureValidator, - remoteVideosValidator, - remoteQaduVideosValidator, - remoteEventsVideosValidator -} from '../../../middlewares' -import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers' -import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib' -import { PodInstance, VideoFileInstance } from '../../../models' -import { - RemoteVideoRequest, - RemoteVideoCreateData, - RemoteVideoUpdateData, - RemoteVideoRemoveData, - RemoteVideoReportAbuseData, - RemoteQaduVideoRequest, - RemoteQaduVideoData, - RemoteVideoEventRequest, - RemoteVideoEventData, - RemoteVideoChannelCreateData, - RemoteVideoChannelUpdateData, - RemoteVideoChannelRemoveData, - RemoteVideoAuthorRemoveData, - RemoteVideoAuthorCreateData -} from '../../../../shared' -import { VideoInstance } from '../../../models/video/video-interface' - -const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] - -// Functions to call when processing a remote request -// FIXME: use RemoteVideoRequestType as id type -const functionsHash: { [ id: string ]: (...args) => Promise } = {} -functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper -functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper - -const remoteVideosRouter = express.Router() - -remoteVideosRouter.post('/', - signatureValidator, - checkSignature, - remoteVideosValidator, - remoteVideos -) - -remoteVideosRouter.post('/qadu', - signatureValidator, - checkSignature, - remoteQaduVideosValidator, - remoteVideosQadu -) - -remoteVideosRouter.post('/events', - signatureValidator, - checkSignature, - remoteEventsVideosValidator, - remoteVideosEvents -) - -// --------------------------------------------------------------------------- - -export { - remoteVideosRouter -} - -// --------------------------------------------------------------------------- - -function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) { - const requests: RemoteVideoRequest[] = req.body.data - const fromPod = res.locals.secure.pod - - // We need to process in the same order to keep consistency - Bluebird.each(requests, request => { - const data = request.data - - // Get the function we need to call in order to process the request - const fun = functionsHash[request.type] - if (fun === undefined) { - logger.error('Unknown remote request type %s.', request.type) - return - } - - return fun.call(this, data, fromPod) - }) - .catch(err => logger.error('Error managing remote videos.', err)) - - // Don't block the other pod - return res.type('json').status(204).end() -} - -function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) { - const requests: RemoteQaduVideoRequest[] = req.body.data - const fromPod = res.locals.secure.pod - - Bluebird.each(requests, request => { - const videoData = request.data - - return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod) - }) - .catch(err => logger.error('Error managing remote videos.', err)) - - return res.type('json').status(204).end() -} - -function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) { - const requests: RemoteVideoEventRequest[] = req.body.data - const fromPod = res.locals.secure.pod - - Bluebird.each(requests, request => { - const eventData = request.data - - return processVideosEventsRetryWrapper(eventData, fromPod) - }) - .catch(err => logger.error('Error managing remote videos.', err)) - - return res.type('json').status(204).end() -} - -async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) { - const options = { - arguments: [ eventData, fromPod ], - errorMessage: 'Cannot process videos events with many retries.' - } - - await retryTransactionWrapper(processVideosEvents, options) -} - -async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) { - await db.sequelize.transaction(async t => { - const sequelizeOptions = { transaction: t } - const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t) - - let columnToUpdate - let qaduType - - switch (eventData.eventType) { - case REQUEST_VIDEO_EVENT_TYPES.VIEWS: - columnToUpdate = 'views' - qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS - break - - case REQUEST_VIDEO_EVENT_TYPES.LIKES: - columnToUpdate = 'likes' - qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES - break - - case REQUEST_VIDEO_EVENT_TYPES.DISLIKES: - columnToUpdate = 'dislikes' - qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES - break - - default: - throw new Error('Unknown video event type.') - } - - const query = {} - query[columnToUpdate] = eventData.count - - await videoInstance.increment(query, sequelizeOptions) - - const qadusParams = [ - { - videoId: videoInstance.id, - type: qaduType - } - ] - await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) - }) - - logger.info('Remote video event processed for video with uuid %s.', eventData.uuid) -} - -async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) { - const options = { - arguments: [ videoData, fromPod ], - errorMessage: 'Cannot update quick and dirty the remote video with many retries.' - } - - await retryTransactionWrapper(quickAndDirtyUpdateVideo, options) -} - -async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { - let videoUUID = '' - - await db.sequelize.transaction(async t => { - const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t) - const sequelizeOptions = { transaction: t } - - videoUUID = videoInstance.uuid - - if (videoData.views) { - videoInstance.set('views', videoData.views) - } - - if (videoData.likes) { - videoInstance.set('likes', videoData.likes) - } - - if (videoData.dislikes) { - videoInstance.set('dislikes', videoData.dislikes) - } - - await videoInstance.save(sequelizeOptions) - }) - - logger.info('Remote video with uuid %s quick and dirty updated', videoUUID) -} - -// Handle retries on fail -async function addRemoteVideoRetryWrapper (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { - const options = { - arguments: [ videoToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video with many retries.' - } - - await retryTransactionWrapper(addRemoteVideo, options) -} - -async function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodInstance) { - logger.debug('Adding remote video "%s".', videoToCreateData.uuid) - - await db.sequelize.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) - if (videoFromDatabase) throw new Error('UUID already exists.') - - const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) - if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') - - const tags = videoToCreateData.tags - const tagInstances = await db.Tag.findOrCreateTags(tags, t) - - const videoData = { - name: videoToCreateData.name, - uuid: videoToCreateData.uuid, - category: videoToCreateData.category, - licence: videoToCreateData.licence, - language: videoToCreateData.language, - nsfw: videoToCreateData.nsfw, - description: videoToCreateData.truncatedDescription, - channelId: videoChannel.id, - duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt, - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: videoToCreateData.updatedAt, - views: videoToCreateData.views, - likes: videoToCreateData.likes, - dislikes: videoToCreateData.dislikes, - remote: true, - privacy: videoToCreateData.privacy - } - - const video = db.Video.build(videoData) - await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) - const videoCreated = await video.save(sequelizeOptions) - - const tasks = [] - for (const fileData of videoToCreateData.files) { - const videoFileInstance = db.VideoFile.build({ - extname: fileData.extname, - infoHash: fileData.infoHash, - resolution: fileData.resolution, - size: fileData.size, - videoId: videoCreated.id - }) - - tasks.push(videoFileInstance.save(sequelizeOptions)) - } - - await Promise.all(tasks) - - await videoCreated.setTags(tagInstances, sequelizeOptions) - }) - - logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) -} - -// Handle retries on fail -async function updateRemoteVideoRetryWrapper (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { - const options = { - arguments: [ videoAttributesToUpdate, fromPod ], - errorMessage: 'Cannot update the remote video with many retries' - } - - await retryTransactionWrapper(updateRemoteVideo, options) -} - -async function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, fromPod: PodInstance) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) - let videoInstance: VideoInstance - let videoFieldsSave: object - - try { - await db.sequelize.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoAttributesToUpdate.uuid, t) - videoFieldsSave = videoInstance.toJSON() - const tags = videoAttributesToUpdate.tags - - const tagInstances = await db.Tag.findOrCreateTags(tags, t) - - videoInstance.set('name', videoAttributesToUpdate.name) - videoInstance.set('category', videoAttributesToUpdate.category) - videoInstance.set('licence', videoAttributesToUpdate.licence) - videoInstance.set('language', videoAttributesToUpdate.language) - videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) - videoInstance.set('description', videoAttributesToUpdate.truncatedDescription) - videoInstance.set('duration', videoAttributesToUpdate.duration) - videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) - videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) - videoInstance.set('views', videoAttributesToUpdate.views) - videoInstance.set('likes', videoAttributesToUpdate.likes) - videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) - videoInstance.set('privacy', videoAttributesToUpdate.privacy) - - await videoInstance.save(sequelizeOptions) - - // Remove old video files - const videoFileDestroyTasks: Bluebird[] = [] - for (const videoFile of videoInstance.VideoFiles) { - videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) - } - await Promise.all(videoFileDestroyTasks) - - const videoFileCreateTasks: Bluebird[] = [] - for (const fileData of videoAttributesToUpdate.files) { - const videoFileInstance = db.VideoFile.build({ - extname: fileData.extname, - infoHash: fileData.infoHash, - resolution: fileData.resolution, - size: fileData.size, - videoId: videoInstance.id - }) - - videoFileCreateTasks.push(videoFileInstance.save(sequelizeOptions)) - } - - await Promise.all(videoFileCreateTasks) - - await videoInstance.setTags(tagInstances, sequelizeOptions) - }) - - logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid) - } catch (err) { - if (videoInstance !== undefined && videoFieldsSave !== undefined) { - resetSequelizeInstance(videoInstance, videoFieldsSave) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', err) - throw err - } -} - -async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { - const options = { - arguments: [ videoToRemoveData, fromPod ], - errorMessage: 'Cannot remove the remote video channel with many retries.' - } - - await retryTransactionWrapper(removeRemoteVideo, options) -} - -async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { - logger.debug('Removing remote video "%s".', videoToRemoveData.uuid) - - await db.sequelize.transaction(async t => { - // We need the instance because we have to remove some other stuffs (thumbnail etc) - const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t) - await videoInstance.destroy({ transaction: t }) - }) - - logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid) -} - -async function addRemoteVideoAuthorRetryWrapper (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) { - const options = { - arguments: [ authorToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video author with many retries.' - } - - await retryTransactionWrapper(addRemoteVideoAuthor, options) -} - -async function addRemoteVideoAuthor (authorToCreateData: RemoteVideoAuthorCreateData, fromPod: PodInstance) { - logger.debug('Adding remote video author "%s".', authorToCreateData.uuid) - - await db.sequelize.transaction(async t => { - const authorInDatabase = await db.Author.loadAuthorByPodAndUUID(authorToCreateData.uuid, fromPod.id, t) - if (authorInDatabase) throw new Error('Author with UUID ' + authorToCreateData.uuid + ' already exists.') - - const videoAuthorData = { - name: authorToCreateData.name, - uuid: authorToCreateData.uuid, - userId: null, // Not on our pod - podId: fromPod.id - } - - const author = db.Author.build(videoAuthorData) - await author.save({ transaction: t }) - }) - - logger.info('Remote video author with uuid %s inserted.', authorToCreateData.uuid) -} - -async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { - const options = { - arguments: [ authorAttributesToRemove, fromPod ], - errorMessage: 'Cannot remove the remote video author with many retries.' - } - - await retryTransactionWrapper(removeRemoteVideoAuthor, options) -} - -async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { - logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid) - - await db.sequelize.transaction(async t => { - const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t) - await videoAuthor.destroy({ transaction: t }) - }) - - logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid) -} - -async function addRemoteVideoChannelRetryWrapper (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) { - const options = { - arguments: [ videoChannelToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video channel with many retries.' - } - - await retryTransactionWrapper(addRemoteVideoChannel, options) -} - -async function addRemoteVideoChannel (videoChannelToCreateData: RemoteVideoChannelCreateData, fromPod: PodInstance) { - logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid) - - await db.sequelize.transaction(async t => { - const videoChannelInDatabase = await db.VideoChannel.loadByUUID(videoChannelToCreateData.uuid) - if (videoChannelInDatabase) { - throw new Error('Video channel with UUID ' + videoChannelToCreateData.uuid + ' already exists.') - } - - const authorUUID = videoChannelToCreateData.ownerUUID - const podId = fromPod.id - - const author = await db.Author.loadAuthorByPodAndUUID(authorUUID, podId, t) - if (!author) throw new Error('Unknown author UUID' + authorUUID + '.') - - const videoChannelData = { - name: videoChannelToCreateData.name, - description: videoChannelToCreateData.description, - uuid: videoChannelToCreateData.uuid, - createdAt: videoChannelToCreateData.createdAt, - updatedAt: videoChannelToCreateData.updatedAt, - remote: true, - authorId: author.id - } - - const videoChannel = db.VideoChannel.build(videoChannelData) - await videoChannel.save({ transaction: t }) - }) - - logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid) -} - -async function updateRemoteVideoChannelRetryWrapper (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) { - const options = { - arguments: [ videoChannelAttributesToUpdate, fromPod ], - errorMessage: 'Cannot update the remote video channel with many retries.' - } - - await retryTransactionWrapper(updateRemoteVideoChannel, options) -} - -async function updateRemoteVideoChannel (videoChannelAttributesToUpdate: RemoteVideoChannelUpdateData, fromPod: PodInstance) { - logger.debug('Updating remote video channel "%s".', videoChannelAttributesToUpdate.uuid) - - await db.sequelize.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const videoChannelInstance = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToUpdate.uuid, t) - videoChannelInstance.set('name', videoChannelAttributesToUpdate.name) - videoChannelInstance.set('description', videoChannelAttributesToUpdate.description) - videoChannelInstance.set('createdAt', videoChannelAttributesToUpdate.createdAt) - videoChannelInstance.set('updatedAt', videoChannelAttributesToUpdate.updatedAt) - - await videoChannelInstance.save(sequelizeOptions) - }) - - logger.info('Remote video channel with uuid %s updated', videoChannelAttributesToUpdate.uuid) -} - -async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { - const options = { - arguments: [ videoChannelAttributesToRemove, fromPod ], - errorMessage: 'Cannot remove the remote video channel with many retries.' - } - - await retryTransactionWrapper(removeRemoteVideoChannel, options) -} - -async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { - logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid) - - await db.sequelize.transaction(async t => { - const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t) - await videoChannel.destroy({ transaction: t }) - }) - - logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid) -} - -async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { - const options = { - arguments: [ reportData, fromPod ], - errorMessage: 'Cannot create remote abuse video with many retries.' - } - - await retryTransactionWrapper(reportAbuseRemoteVideo, options) -} - -async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { - logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID) - - await db.sequelize.transaction(async t => { - const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t) - const videoAbuseData = { - reporterUsername: reportData.reporterUsername, - reason: reportData.reportReason, - reporterPodId: fromPod.id, - videoId: videoInstance.id - } - - await db.VideoAbuse.create(videoAbuseData) - - }) - - logger.info('Remote abuse for video uuid %s created', reportData.videoUUID) -} - -async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) { - try { - const video = await db.Video.loadLocalVideoByUUID(id, t) - - if (!video) throw new Error('Video ' + id + ' not found') - - return video - } catch (err) { - logger.error('Cannot load owned video from id.', { error: err.stack, id }) - throw err - } -} - -async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) { - try { - const video = await db.Video.loadByHostAndUUID(podHost, uuid, t) - if (!video) throw new Error('Video not found') - - return video - } catch (err) { - logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid }) - throw err - } -} +// import * as express from 'express' +// import * as Bluebird from 'bluebird' +// import * as Sequelize from 'sequelize' +// +// import { database as db } from '../../../initializers/database' +// import { +// REQUEST_ENDPOINT_ACTIONS, +// REQUEST_ENDPOINTS, +// REQUEST_VIDEO_EVENT_TYPES, +// REQUEST_VIDEO_QADU_TYPES +// } from '../../../initializers' +// import { +// checkSignature, +// signatureValidator, +// remoteVideosValidator, +// remoteQaduVideosValidator, +// remoteEventsVideosValidator +// } from '../../../middlewares' +// import { logger, retryTransactionWrapper, resetSequelizeInstance } from '../../../helpers' +// import { quickAndDirtyUpdatesVideoToFriends, fetchVideoChannelByHostAndUUID } from '../../../lib' +// import { PodInstance, VideoFileInstance } from '../../../models' +// import { +// RemoteVideoRequest, +// RemoteVideoCreateData, +// RemoteVideoUpdateData, +// RemoteVideoRemoveData, +// RemoteVideoReportAbuseData, +// RemoteQaduVideoRequest, +// RemoteQaduVideoData, +// RemoteVideoEventRequest, +// RemoteVideoEventData, +// RemoteVideoChannelCreateData, +// RemoteVideoChannelUpdateData, +// RemoteVideoChannelRemoveData, +// RemoteVideoAuthorRemoveData, +// RemoteVideoAuthorCreateData +// } from '../../../../shared' +// import { VideoInstance } from '../../../models/video/video-interface' +// +// const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] +// +// // Functions to call when processing a remote request +// // FIXME: use RemoteVideoRequestType as id type +// const functionsHash: { [ id: string ]: (...args) => Promise } = {} +// functionsHash[ENDPOINT_ACTIONS.ADD_VIDEO] = addRemoteVideoRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.UPDATE_VIDEO] = updateRemoteVideoRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.REMOVE_VIDEO] = removeRemoteVideoRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.ADD_CHANNEL] = addRemoteVideoChannelRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = updateRemoteVideoChannelRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = removeRemoteVideoChannelRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideoRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.ADD_AUTHOR] = addRemoteVideoAuthorRetryWrapper +// functionsHash[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = removeRemoteVideoAuthorRetryWrapper +// +// const remoteVideosRouter = express.Router() +// +// remoteVideosRouter.post('/', +// signatureValidator, +// checkSignature, +// remoteVideosValidator, +// remoteVideos +// ) +// +// remoteVideosRouter.post('/qadu', +// signatureValidator, +// checkSignature, +// remoteQaduVideosValidator, +// remoteVideosQadu +// ) +// +// remoteVideosRouter.post('/events', +// signatureValidator, +// checkSignature, +// remoteEventsVideosValidator, +// remoteVideosEvents +// ) +// +// // --------------------------------------------------------------------------- +// +// export { +// remoteVideosRouter +// } +// +// // --------------------------------------------------------------------------- +// +// function remoteVideos (req: express.Request, res: express.Response, next: express.NextFunction) { +// const requests: RemoteVideoRequest[] = req.body.data +// const fromPod = res.locals.secure.pod +// +// // We need to process in the same order to keep consistency +// Bluebird.each(requests, request => { +// const data = request.data +// +// // Get the function we need to call in order to process the request +// const fun = functionsHash[request.type] +// if (fun === undefined) { +// logger.error('Unknown remote request type %s.', request.type) +// return +// } +// +// return fun.call(this, data, fromPod) +// }) +// .catch(err => logger.error('Error managing remote videos.', err)) +// +// // Don't block the other pod +// return res.type('json').status(204).end() +// } +// +// function remoteVideosQadu (req: express.Request, res: express.Response, next: express.NextFunction) { +// const requests: RemoteQaduVideoRequest[] = req.body.data +// const fromPod = res.locals.secure.pod +// +// Bluebird.each(requests, request => { +// const videoData = request.data +// +// return quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod) +// }) +// .catch(err => logger.error('Error managing remote videos.', err)) +// +// return res.type('json').status(204).end() +// } +// +// function remoteVideosEvents (req: express.Request, res: express.Response, next: express.NextFunction) { +// const requests: RemoteVideoEventRequest[] = req.body.data +// const fromPod = res.locals.secure.pod +// +// Bluebird.each(requests, request => { +// const eventData = request.data +// +// return processVideosEventsRetryWrapper(eventData, fromPod) +// }) +// .catch(err => logger.error('Error managing remote videos.', err)) +// +// return res.type('json').status(204).end() +// } +// +// async function processVideosEventsRetryWrapper (eventData: RemoteVideoEventData, fromPod: PodInstance) { +// const options = { +// arguments: [ eventData, fromPod ], +// errorMessage: 'Cannot process videos events with many retries.' +// } +// +// await retryTransactionWrapper(processVideosEvents, options) +// } +// +// async function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInstance) { +// await db.sequelize.transaction(async t => { +// const sequelizeOptions = { transaction: t } +// const videoInstance = await fetchLocalVideoByUUID(eventData.uuid, t) +// +// let columnToUpdate +// let qaduType +// +// switch (eventData.eventType) { +// case REQUEST_VIDEO_EVENT_TYPES.VIEWS: +// columnToUpdate = 'views' +// qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS +// break +// +// case REQUEST_VIDEO_EVENT_TYPES.LIKES: +// columnToUpdate = 'likes' +// qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES +// break +// +// case REQUEST_VIDEO_EVENT_TYPES.DISLIKES: +// columnToUpdate = 'dislikes' +// qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES +// break +// +// default: +// throw new Error('Unknown video event type.') +// } +// +// const query = {} +// query[columnToUpdate] = eventData.count +// +// await videoInstance.increment(query, sequelizeOptions) +// +// const qadusParams = [ +// { +// videoId: videoInstance.id, +// type: qaduType +// } +// ] +// await quickAndDirtyUpdatesVideoToFriends(qadusParams, t) +// }) +// +// logger.info('Remote video event processed for video with uuid %s.', eventData.uuid) +// } +// +// async function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, fromPod: PodInstance) { +// const options = { +// arguments: [ videoData, fromPod ], +// errorMessage: 'Cannot update quick and dirty the remote video with many retries.' +// } +// +// await retryTransactionWrapper(quickAndDirtyUpdateVideo, options) +// } +// +// async function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) { +// let videoUUID = '' +// +// await db.sequelize.transaction(async t => { +// const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoData.uuid, t) +// const sequelizeOptions = { transaction: t } +// +// videoUUID = videoInstance.uuid +// +// if (videoData.views) { +// videoInstance.set('views', videoData.views) +// } +// +// if (videoData.likes) { +// videoInstance.set('likes', videoData.likes) +// } +// +// if (videoData.dislikes) { +// videoInstance.set('dislikes', videoData.dislikes) +// } +// +// await videoInstance.save(sequelizeOptions) +// }) +// +// logger.info('Remote video with uuid %s quick and dirty updated', videoUUID) +// } +// +// async function removeRemoteVideoRetryWrapper (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { +// const options = { +// arguments: [ videoToRemoveData, fromPod ], +// errorMessage: 'Cannot remove the remote video channel with many retries.' +// } +// +// await retryTransactionWrapper(removeRemoteVideo, options) +// } +// +// async function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: PodInstance) { +// logger.debug('Removing remote video "%s".', videoToRemoveData.uuid) +// +// await db.sequelize.transaction(async t => { +// // We need the instance because we have to remove some other stuffs (thumbnail etc) +// const videoInstance = await fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid, t) +// await videoInstance.destroy({ transaction: t }) +// }) +// +// logger.info('Remote video with uuid %s removed.', videoToRemoveData.uuid) +// } +// +// async function removeRemoteVideoAuthorRetryWrapper (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { +// const options = { +// arguments: [ authorAttributesToRemove, fromPod ], +// errorMessage: 'Cannot remove the remote video author with many retries.' +// } +// +// await retryTransactionWrapper(removeRemoteVideoAuthor, options) +// } +// +// async function removeRemoteVideoAuthor (authorAttributesToRemove: RemoteVideoAuthorRemoveData, fromPod: PodInstance) { +// logger.debug('Removing remote video author "%s".', authorAttributesToRemove.uuid) +// +// await db.sequelize.transaction(async t => { +// const videoAuthor = await db.Author.loadAuthorByPodAndUUID(authorAttributesToRemove.uuid, fromPod.id, t) +// await videoAuthor.destroy({ transaction: t }) +// }) +// +// logger.info('Remote video author with uuid %s removed.', authorAttributesToRemove.uuid) +// } +// +// async function removeRemoteVideoChannelRetryWrapper (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { +// const options = { +// arguments: [ videoChannelAttributesToRemove, fromPod ], +// errorMessage: 'Cannot remove the remote video channel with many retries.' +// } +// +// await retryTransactionWrapper(removeRemoteVideoChannel, options) +// } +// +// async function removeRemoteVideoChannel (videoChannelAttributesToRemove: RemoteVideoChannelRemoveData, fromPod: PodInstance) { +// logger.debug('Removing remote video channel "%s".', videoChannelAttributesToRemove.uuid) +// +// await db.sequelize.transaction(async t => { +// const videoChannel = await fetchVideoChannelByHostAndUUID(fromPod.host, videoChannelAttributesToRemove.uuid, t) +// await videoChannel.destroy({ transaction: t }) +// }) +// +// logger.info('Remote video channel with uuid %s removed.', videoChannelAttributesToRemove.uuid) +// } +// +// async function reportAbuseRemoteVideoRetryWrapper (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { +// const options = { +// arguments: [ reportData, fromPod ], +// errorMessage: 'Cannot create remote abuse video with many retries.' +// } +// +// await retryTransactionWrapper(reportAbuseRemoteVideo, options) +// } +// +// async function reportAbuseRemoteVideo (reportData: RemoteVideoReportAbuseData, fromPod: PodInstance) { +// logger.debug('Reporting remote abuse for video %s.', reportData.videoUUID) +// +// await db.sequelize.transaction(async t => { +// const videoInstance = await fetchLocalVideoByUUID(reportData.videoUUID, t) +// const videoAbuseData = { +// reporterUsername: reportData.reporterUsername, +// reason: reportData.reportReason, +// reporterPodId: fromPod.id, +// videoId: videoInstance.id +// } +// +// await db.VideoAbuse.create(videoAbuseData) +// +// }) +// +// logger.info('Remote abuse for video uuid %s created', reportData.videoUUID) +// } +// +// async function fetchLocalVideoByUUID (id: string, t: Sequelize.Transaction) { +// try { +// const video = await db.Video.loadLocalVideoByUUID(id, t) +// +// if (!video) throw new Error('Video ' + id + ' not found') +// +// return video +// } catch (err) { +// logger.error('Cannot load owned video from id.', { error: err.stack, id }) +// throw err +// } +// } +// +// async function fetchVideoByHostAndUUID (podHost: string, uuid: string, t: Sequelize.Transaction) { +// try { +// const video = await db.Video.loadByHostAndUUID(podHost, uuid, t) +// if (!video) throw new Error('Video not found') +// +// return video +// } catch (err) { +// logger.error('Cannot load video from host and uuid.', { error: err.stack, podHost, uuid }) +// throw err +// } +// } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 4dd09917b..964db151d 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -10,7 +10,8 @@ import { VIDEO_CATEGORIES, VIDEO_LICENCES, VIDEO_LANGUAGES, - VIDEO_PRIVACIES + VIDEO_PRIVACIES, + VIDEO_MIMETYPE_EXT } from '../../../initializers' import { addEventToRemoteVideo, @@ -50,6 +51,7 @@ import { abuseVideoRouter } from './abuse' import { blacklistRouter } from './blacklist' import { rateVideoRouter } from './rate' import { videoChannelRouter } from './channel' +import { getActivityPubUrl } from '../../../helpers/activitypub' const videosRouter = express.Router() @@ -59,19 +61,18 @@ const storage = multer.diskStorage({ cb(null, CONFIG.STORAGE.VIDEOS_DIR) }, - filename: (req, file, cb) => { - let extension = '' - if (file.mimetype === 'video/webm') extension = 'webm' - else if (file.mimetype === 'video/mp4') extension = 'mp4' - else if (file.mimetype === 'video/ogg') extension = 'ogv' - generateRandomString(16) - .then(randomString => { - cb(null, randomString + '.' + extension) - }) - .catch(err => { - logger.error('Cannot generate random string for file name.', err) - throw err - }) + filename: async (req, file, cb) => { + const extension = VIDEO_MIMETYPE_EXT[file.mimetype] + let randomString = '' + + try { + randomString = await generateRandomString(16) + } catch (err) { + logger.error('Cannot generate random string for file name.', err) + randomString = 'fake-random-string' + } + + cb(null, randomString + '.' + extension) } }) @@ -190,6 +191,7 @@ async function addVideo (req: express.Request, res: express.Response, videoPhysi channelId: res.locals.videoChannel.id } const video = db.Video.build(videoData) + video.url = getActivityPubUrl('video', video.uuid) const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename) const videoFileHeight = await getVideoFileHeight(videoFilePath) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index ecb509b66..75de2278c 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -2,10 +2,48 @@ import * as url from 'url' import { database as db } from '../initializers' import { logger } from './logger' -import { doRequest } from './requests' +import { doRequest, doRequestAndSaveToFile } from './requests' import { isRemoteAccountValid } from './custom-validators' import { ActivityPubActor } from '../../shared/models/activitypub/activitypub-actor' import { ResultList } from '../../shared/models/result-list.model' +import { CONFIG } from '../initializers/constants' +import { VideoInstance } from '../models/video/video-interface' +import { ActivityIconObject } from '../../shared/index' +import { join } from 'path' + +function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) { + const thumbnailName = video.getThumbnailName() + const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) + + const options = { + method: 'GET', + uri: icon.url + } + return doRequestAndSaveToFile(options, thumbnailPath) +} + +function getActivityPubUrl (type: 'video' | 'videoChannel', uuid: string) { + if (type === 'video') return CONFIG.WEBSERVER.URL + '/videos/watch/' + uuid + else if (type === 'videoChannel') return CONFIG.WEBSERVER.URL + '/video-channels/' + uuid + + return '' +} + +async function getOrCreateAccount (accountUrl: string) { + let account = await db.Account.loadByUrl(accountUrl) + + // We don't have this account in our database, fetch it on remote + if (!account) { + const { account } = await fetchRemoteAccountAndCreatePod(accountUrl) + + if (!account) throw new Error('Cannot fetch remote account.') + + // Save our new account in database + await account.save() + } + + return account +} async function fetchRemoteAccountAndCreatePod (accountUrl: string) { const options = { @@ -100,7 +138,10 @@ function activityPubCollectionPagination (url: string, page: number, result: Res export { fetchRemoteAccountAndCreatePod, activityPubContextify, - activityPubCollectionPagination + activityPubCollectionPagination, + getActivityPubUrl, + generateThumbnailFromUrl, + getOrCreateAccount } // --------------------------------------------------------------------------- diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts new file mode 100644 index 000000000..dd671c4cf --- /dev/null +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -0,0 +1,34 @@ +import * as validator from 'validator' +import { + isVideoChannelCreateActivityValid, + isVideoTorrentAddActivityValid, + isVideoTorrentUpdateActivityValid, + isVideoChannelUpdateActivityValid +} from './videos' + +function isRootActivityValid (activity: any) { + return Array.isArray(activity['@context']) && + ( + (activity.type === 'Collection' || activity.type === 'OrderedCollection') && + validator.isInt(activity.totalItems, { min: 0 }) && + Array.isArray(activity.items) + ) || + ( + validator.isURL(activity.id) && + validator.isURL(activity.actor) + ) +} + +function isActivityValid (activity: any) { + return isVideoTorrentAddActivityValid(activity) || + isVideoChannelCreateActivityValid(activity) || + isVideoTorrentUpdateActivityValid(activity) || + isVideoChannelUpdateActivityValid(activity) +} + +// --------------------------------------------------------------------------- + +export { + isRootActivityValid, + isActivityValid +} diff --git a/server/helpers/custom-validators/activitypub/index.ts b/server/helpers/custom-validators/activitypub/index.ts index 800f0ddf3..0eba06a7b 100644 --- a/server/helpers/custom-validators/activitypub/index.ts +++ b/server/helpers/custom-validators/activitypub/index.ts @@ -1,4 +1,5 @@ export * from './account' +export * from './activity' export * from './signature' export * from './misc' export * from './videos' diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 806d33483..f049f5a8c 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -12,6 +12,16 @@ function isActivityPubUrlValid (url: string) { return exists(url) && validator.isURL(url, isURLOptions) } +function isBaseActivityValid (activity: any, type: string) { + return Array.isArray(activity['@context']) && + activity.type === type && + validator.isURL(activity.id) && + validator.isURL(activity.actor) && + Array.isArray(activity.to) && + activity.to.every(t => validator.isURL(t)) +} + export { - isActivityPubUrlValid + isActivityPubUrlValid, + isBaseActivityValid } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index e0ffba679..9233a1359 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -1,184 +1,117 @@ -import 'express-validator' -import { has, values } from 'lodash' +import * as validator from 'validator' import { - REQUEST_ENDPOINTS, - REQUEST_ENDPOINT_ACTIONS, - REQUEST_VIDEO_EVENT_TYPES + ACTIVITY_PUB } from '../../../initializers' -import { isArray, isDateValid, isUUIDValid } from '../misc' +import { isDateValid, isUUIDValid } from '../misc' import { - isVideoThumbnailDataValid, - isVideoAbuseReasonValid, - isVideoAbuseReporterUsernameValid, isVideoViewsValid, - isVideoLikesValid, - isVideoDislikesValid, - isVideoEventCountValid, - isRemoteVideoCategoryValid, - isRemoteVideoLicenceValid, - isRemoteVideoLanguageValid, isVideoNSFWValid, isVideoTruncatedDescriptionValid, isVideoDurationValid, - isVideoFileInfoHashValid, isVideoNameValid, - isVideoTagsValid, - isVideoFileExtnameValid, - isVideoFileResolutionValid + isVideoTagValid } from '../videos' import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../video-channels' -import { isVideoAuthorNameValid } from '../video-authors' - -const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] - -const checkers: { [ id: string ]: (obj: any) => boolean } = {} -checkers[ENDPOINT_ACTIONS.ADD_VIDEO] = checkAddVideo -checkers[ENDPOINT_ACTIONS.UPDATE_VIDEO] = checkUpdateVideo -checkers[ENDPOINT_ACTIONS.REMOVE_VIDEO] = checkRemoveVideo -checkers[ENDPOINT_ACTIONS.REPORT_ABUSE] = checkReportVideo -checkers[ENDPOINT_ACTIONS.ADD_CHANNEL] = checkAddVideoChannel -checkers[ENDPOINT_ACTIONS.UPDATE_CHANNEL] = checkUpdateVideoChannel -checkers[ENDPOINT_ACTIONS.REMOVE_CHANNEL] = checkRemoveVideoChannel -checkers[ENDPOINT_ACTIONS.ADD_AUTHOR] = checkAddAuthor -checkers[ENDPOINT_ACTIONS.REMOVE_AUTHOR] = checkRemoveAuthor - -function removeBadRequestVideos (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const video = request.data - - if ( - !video || - checkers[request.type] === undefined || - checkers[request.type](video) === false - ) { - requests.splice(i, 1) - } - } +import { isBaseActivityValid } from './misc' + +function isVideoTorrentAddActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Add') && + isVideoTorrentObjectValid(activity.object) +} + +function isVideoTorrentUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + isVideoTorrentObjectValid(activity.object) } -function removeBadRequestVideosQadu (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const video = request.data - - if ( - !video || - ( - isUUIDValid(video.uuid) && - (has(video, 'views') === false || isVideoViewsValid(video.views)) && - (has(video, 'likes') === false || isVideoLikesValid(video.likes)) && - (has(video, 'dislikes') === false || isVideoDislikesValid(video.dislikes)) - ) === false - ) { - requests.splice(i, 1) - } - } +function isVideoTorrentObjectValid (video: any) { + return video.type === 'Video' && + isVideoNameValid(video.name) && + isVideoDurationValid(video.duration) && + isUUIDValid(video.uuid) && + setValidRemoteTags(video) && + isRemoteIdentifierValid(video.category) && + isRemoteIdentifierValid(video.licence) && + isRemoteIdentifierValid(video.language) && + isVideoViewsValid(video.video) && + isVideoNSFWValid(video.nsfw) && + isDateValid(video.published) && + isDateValid(video.updated) && + isRemoteVideoContentValid(video.mediaType, video.content) && + isRemoteVideoIconValid(video.icon) && + setValidRemoteVideoUrls(video.url) } -function removeBadRequestVideosEvents (requests: any[]) { - for (let i = requests.length - 1; i >= 0 ; i--) { - const request = requests[i] - const eventData = request.data - - if ( - !eventData || - ( - isUUIDValid(eventData.uuid) && - values(REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && - isVideoEventCountValid(eventData.count) - ) === false - ) { - requests.splice(i, 1) - } - } +function isVideoChannelCreateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Create') && + isVideoChannelObjectValid(activity.object) +} + +function isVideoChannelUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + isVideoChannelObjectValid(activity.object) +} + +function isVideoChannelObjectValid (videoChannel: any) { + return videoChannel.type === 'VideoChannel' && + isVideoChannelNameValid(videoChannel.name) && + isVideoChannelDescriptionValid(videoChannel.description) && + isUUIDValid(videoChannel.uuid) } // --------------------------------------------------------------------------- export { - removeBadRequestVideos, - removeBadRequestVideosQadu, - removeBadRequestVideosEvents + isVideoTorrentAddActivityValid, + isVideoChannelCreateActivityValid, + isVideoTorrentUpdateActivityValid, + isVideoChannelUpdateActivityValid } // --------------------------------------------------------------------------- -function isCommonVideoAttributesValid (video: any) { - return isDateValid(video.createdAt) && - isDateValid(video.updatedAt) && - isRemoteVideoCategoryValid(video.category) && - isRemoteVideoLicenceValid(video.licence) && - isRemoteVideoLanguageValid(video.language) && - isVideoNSFWValid(video.nsfw) && - isVideoTruncatedDescriptionValid(video.truncatedDescription) && - isVideoDurationValid(video.duration) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isUUIDValid(video.uuid) && - isVideoViewsValid(video.views) && - isVideoLikesValid(video.likes) && - isVideoDislikesValid(video.dislikes) && - isArray(video.files) && - video.files.every(videoFile => { - if (!videoFile) return false - - return ( - isVideoFileInfoHashValid(videoFile.infoHash) && - isVideoFileExtnameValid(videoFile.extname) && - isVideoFileResolutionValid(videoFile.resolution) - ) - }) -} +function setValidRemoteTags (video: any) { + if (Array.isArray(video.tag) === false) return false -function checkAddVideo (video: any) { - return isCommonVideoAttributesValid(video) && - isUUIDValid(video.channelUUID) && - isVideoThumbnailDataValid(video.thumbnailData) -} + const newTag = video.tag.filter(t => { + return t.type === 'Hashtag' && + isVideoTagValid(t.name) + }) -function checkUpdateVideo (video: any) { - return isCommonVideoAttributesValid(video) + video.tag = newTag + return true } -function checkRemoveVideo (video: any) { - return isUUIDValid(video.uuid) +function isRemoteIdentifierValid (data: any) { + return validator.isInt(data.identifier, { min: 0 }) } -function checkReportVideo (abuse: any) { - return isUUIDValid(abuse.videoUUID) && - isVideoAbuseReasonValid(abuse.reportReason) && - isVideoAbuseReporterUsernameValid(abuse.reporterUsername) +function isRemoteVideoContentValid (mediaType: string, content: string) { + return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) } -function checkAddVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) && - isVideoChannelNameValid(videoChannel.name) && - isVideoChannelDescriptionValid(videoChannel.description) && - isDateValid(videoChannel.createdAt) && - isDateValid(videoChannel.updatedAt) && - isUUIDValid(videoChannel.ownerUUID) +function isRemoteVideoIconValid (icon: any) { + return icon.type === 'Image' && + validator.isURL(icon.url) && + icon.mediaType === 'image/jpeg' && + validator.isInt(icon.width, { min: 0 }) && + validator.isInt(icon.height, { min: 0 }) } -function checkUpdateVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) && - isVideoChannelNameValid(videoChannel.name) && - isVideoChannelDescriptionValid(videoChannel.description) && - isDateValid(videoChannel.createdAt) && - isDateValid(videoChannel.updatedAt) && - isUUIDValid(videoChannel.ownerUUID) -} +function setValidRemoteVideoUrls (video: any) { + if (Array.isArray(video.url) === false) return false -function checkRemoveVideoChannel (videoChannel: any) { - return isUUIDValid(videoChannel.uuid) -} + const newUrl = video.url.filter(u => isRemoteVideoUrlValid(u)) + video.url = newUrl -function checkAddAuthor (author: any) { - return isUUIDValid(author.uuid) && - isVideoAuthorNameValid(author.name) + return true } -function checkRemoveAuthor (author: any) { - return isUUIDValid(author.uuid) +function isRemoteVideoUrlValid (url: any) { + return url.type === 'Link' && + ACTIVITY_PUB.VIDEO_URL_MIME_TYPES.indexOf(url.mimeType) !== -1 && + validator.isURL(url.url) && + validator.isInt(url.width, { min: 0 }) && + validator.isInt(url.size, { min: 0 }) } diff --git a/server/helpers/custom-validators/index.ts b/server/helpers/custom-validators/index.ts index 869b08870..58a40249b 100644 --- a/server/helpers/custom-validators/index.ts +++ b/server/helpers/custom-validators/index.ts @@ -3,6 +3,5 @@ export * from './misc' export * from './pods' export * from './pods' export * from './users' -export * from './video-authors' export * from './video-channels' export * from './videos' diff --git a/server/helpers/custom-validators/video-authors.ts b/server/helpers/custom-validators/video-authors.ts deleted file mode 100644 index 48ca9b200..000000000 --- a/server/helpers/custom-validators/video-authors.ts +++ /dev/null @@ -1,45 +0,0 @@ -import * as Promise from 'bluebird' -import * as validator from 'validator' -import * as express from 'express' -import 'express-validator' - -import { database as db } from '../../initializers' -import { AuthorInstance } from '../../models' -import { logger } from '../logger' - -import { isUserUsernameValid } from './users' - -function isVideoAuthorNameValid (value: string) { - return isUserUsernameValid(value) -} - -function checkVideoAuthorExists (id: string, res: express.Response, callback: () => void) { - let promise: Promise - if (validator.isInt(id)) { - promise = db.Author.load(+id) - } else { // UUID - promise = db.Author.loadByUUID(id) - } - - promise.then(author => { - if (!author) { - return res.status(404) - .json({ error: 'Video author not found' }) - .end() - } - - res.locals.author = author - callback() - }) - .catch(err => { - logger.error('Error in video author request validator.', err) - return res.sendStatus(500) - }) -} - -// --------------------------------------------------------------------------- - -export { - checkVideoAuthorExists, - isVideoAuthorNameValid -} diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index f3fdcaf2d..83407f17b 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -73,19 +73,26 @@ function isVideoDescriptionValid (value: string) { } function isVideoDurationValid (value: string) { - return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return exists(value) && + typeof value === 'string' && + value.startsWith('PT') && + value.endsWith('S') && + validator.isInt(value.replace(/[^0-9]+/, ''), VIDEOS_CONSTRAINTS_FIELDS.DURATION) } function isVideoNameValid (value: string) { return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) } +function isVideoTagValid (tag: string) { + return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) +} + function isVideoTagsValid (tags: string[]) { return isArray(tags) && validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) && - tags.every(tag => { - return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) - }) + tags.every(tag => isVideoTagValid(tag)) } function isVideoThumbnailValid (value: string) { @@ -209,6 +216,7 @@ export { isRemoteVideoPrivacyValid, isVideoFileResolutionValid, checkVideoExists, + isVideoTagValid, isRemoteVideoCategoryValid, isRemoteVideoLicenceValid, isRemoteVideoLanguageValid diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 8c4c983f7..31cedd768 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -10,6 +10,7 @@ import { import { PodInstance } from '../models' import { PodSignature } from '../../shared' import { signObject } from './peertube-crypto' +import { createWriteStream } from 'fs' function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { @@ -17,6 +18,15 @@ function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { }) } +function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.UriOptions, destPath: string) { + return new Promise((res, rej) => { + request(requestOptions) + .on('response', response => res(response as request.RequestResponse)) + .on('error', err => rej(err)) + .pipe(createWriteStream(destPath)) + }) +} + type MakeRetryRequestParams = { url: string, method: 'GET' | 'POST', @@ -88,6 +98,7 @@ function makeSecureRequest (params: MakeSecureRequestParams) { export { doRequest, + doRequestAndSaveToFile, makeRetryRequest, makeSecureRequest } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index cb838cf16..e1f877e80 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -203,6 +203,12 @@ const VIDEO_PRIVACIES = { [VideoPrivacy.PRIVATE]: 'Private' } +const VIDEO_MIMETYPE_EXT = { + 'video/webm': 'webm', + 'video/ogg': 'ogv', + 'video/mp4': 'mp4' +} + // --------------------------------------------------------------------------- // Score a pod has when we create it as a friend @@ -212,7 +218,14 @@ const FRIEND_SCORE = { } const ACTIVITY_PUB = { - COLLECTION_ITEMS_PER_PAGE: 10 + COLLECTION_ITEMS_PER_PAGE: 10, + VIDEO_URL_MIME_TYPES: [ + 'video/mp4', + 'video/webm', + 'video/ogg', + 'application/x-bittorrent', + 'application/x-bittorrent;x-scheme-handler/magnet' + ] } // --------------------------------------------------------------------------- @@ -245,42 +258,6 @@ const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50 // Number of requests to retry for replay requests module const RETRY_REQUESTS = 5 -const REQUEST_ENDPOINTS: { [ id: string ]: RequestEndpoint } = { - VIDEOS: 'videos' -} - -const REQUEST_ENDPOINT_ACTIONS: { - [ id: string ]: { - [ id: string ]: RemoteVideoRequestType - } -} = {} -REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { - ADD_VIDEO: 'add-video', - UPDATE_VIDEO: 'update-video', - REMOVE_VIDEO: 'remove-video', - ADD_CHANNEL: 'add-channel', - UPDATE_CHANNEL: 'update-channel', - REMOVE_CHANNEL: 'remove-channel', - ADD_AUTHOR: 'add-author', - REMOVE_AUTHOR: 'remove-author', - REPORT_ABUSE: 'report-abuse' -} - -const REQUEST_VIDEO_QADU_ENDPOINT = 'videos/qadu' -const REQUEST_VIDEO_EVENT_ENDPOINT = 'videos/events' - -const REQUEST_VIDEO_QADU_TYPES: { [ id: string ]: RequestVideoQaduType } = { - LIKES: 'likes', - DISLIKES: 'dislikes', - VIEWS: 'views' -} - -const REQUEST_VIDEO_EVENT_TYPES: { [ id: string ]: RequestVideoEventType } = { - LIKES: 'likes', - DISLIKES: 'dislikes', - VIEWS: 'views' -} - const REMOTE_SCHEME = { HTTP: 'https', WS: 'wss' @@ -306,8 +283,6 @@ let JOBS_FETCHING_INTERVAL = 60000 // --------------------------------------------------------------------------- -// const SIGNATURE_ALGORITHM = 'RSA-SHA256' -// const SIGNATURE_ENCODING = 'hex' const PRIVATE_RSA_KEY_SIZE = 2048 // Password encryption @@ -412,5 +387,6 @@ export { VIDEO_LANGUAGES, VIDEO_PRIVACIES, VIDEO_LICENCES, - VIDEO_RATE_TYPES + VIDEO_RATE_TYPES, + VIDEO_MIMETYPE_EXT } diff --git a/server/lib/activitypub/misc.ts b/server/lib/activitypub/misc.ts new file mode 100644 index 000000000..05e77ebc3 --- /dev/null +++ b/server/lib/activitypub/misc.ts @@ -0,0 +1,77 @@ +import * as magnetUtil from 'magnet-uri' +import * as Sequelize from 'sequelize' +import { VideoTorrentObject } from '../../../shared' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { database as db } from '../../initializers' +import { VIDEO_MIMETYPE_EXT } from '../../initializers/constants' +import { VideoChannelInstance } from '../../models/video/video-channel-interface' +import { VideoFileAttributes } from '../../models/video/video-file-interface' +import { VideoAttributes, VideoInstance } from '../../models/video/video-interface' + +async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelInstance, videoObject: VideoTorrentObject, t: Sequelize.Transaction) { + const videoFromDatabase = await db.Video.loadByUUIDOrURL(videoObject.uuid, videoObject.id, t) + if (videoFromDatabase) throw new Error('Video with this UUID/Url already exists.') + + const duration = videoObject.duration.replace(/[^\d]+/, '') + const videoData: VideoAttributes = { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category: parseInt(videoObject.category.identifier, 10), + licence: parseInt(videoObject.licence.identifier, 10), + language: parseInt(videoObject.language.identifier, 10), + nsfw: videoObject.nsfw, + description: videoObject.content, + channelId: videoChannel.id, + duration: parseInt(duration, 10), + createdAt: videoObject.published, + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: videoObject.updated, + views: videoObject.views, + likes: 0, + dislikes: 0, + // likes: videoToCreateData.likes, + // dislikes: videoToCreateData.dislikes, + remote: true, + privacy: 1 + // privacy: videoToCreateData.privacy + } + + return videoData +} + +function videoFileActivityUrlToDBAttributes (videoCreated: VideoInstance, videoObject: VideoTorrentObject) { + const fileUrls = videoObject.url + .filter(u => Object.keys(VIDEO_MIMETYPE_EXT).indexOf(u.mimeType) !== -1) + + const attributes: VideoFileAttributes[] = [] + for (const url of fileUrls) { + // Fetch associated magnet uri + const magnet = videoObject.url + .find(u => { + return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === url.width + }) + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + url.url) + + const parsed = magnetUtil.decode(magnet.url) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url) + + const attribute = { + extname: VIDEO_MIMETYPE_EXT[url.mimeType], + infoHash: parsed.infoHash, + resolution: url.width, + size: url.size, + videoId: videoCreated.id + } + attributes.push(attribute) + } + + return attributes +} + +// --------------------------------------------------------------------------- + +export { + videoFileActivityUrlToDBAttributes, + videoActivityObjectToDBAttributes +} diff --git a/server/lib/activitypub/process-add.ts b/server/lib/activitypub/process-add.ts new file mode 100644 index 000000000..40541aca3 --- /dev/null +++ b/server/lib/activitypub/process-add.ts @@ -0,0 +1,72 @@ +import { VideoTorrentObject } from '../../../shared' +import { ActivityAdd } from '../../../shared/models/activitypub/activity' +import { generateThumbnailFromUrl, logger, retryTransactionWrapper, getOrCreateAccount } from '../../helpers' +import { database as db } from '../../initializers' +import { AccountInstance } from '../../models/account/account-interface' +import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import Bluebird = require('bluebird') + +async function processAddActivity (activity: ActivityAdd) { + const activityObject = activity.object + const activityType = activityObject.type + const account = await getOrCreateAccount(activity.actor) + + if (activityType === 'Video') { + return processAddVideo(account, activity.id, activityObject as VideoTorrentObject) + } + + logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) + return Promise.resolve(undefined) +} + +// --------------------------------------------------------------------------- + +export { + processAddActivity +} + +// --------------------------------------------------------------------------- + +function processAddVideo (account: AccountInstance, videoChannelUrl: string, video: VideoTorrentObject) { + const options = { + arguments: [ account, videoChannelUrl ,video ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + return retryTransactionWrapper(addRemoteVideo, options) +} + +async function addRemoteVideo (account: AccountInstance, videoChannelUrl: string, videoToCreateData: VideoTorrentObject) { + logger.debug('Adding remote video %s.', videoToCreateData.url) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoChannel = await db.VideoChannel.loadByUrl(videoChannelUrl, t) + if (!videoChannel) throw new Error('Video channel not found.') + + if (videoChannel.Account.id !== account.id) throw new Error('Video channel is not owned by this account.') + + const videoData = await videoActivityObjectToDBAttributes(videoChannel, videoToCreateData, t) + const video = db.Video.build(videoData) + + // Don't block on request + generateThumbnailFromUrl(video, videoToCreateData.icon) + .catch(err => logger.warning('Cannot generate thumbnail of %s.', videoToCreateData.id, err)) + + const videoCreated = await video.save(sequelizeOptions) + + const videoFileAttributes = await videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData) + + const tasks: Bluebird[] = videoFileAttributes.map(f => db.VideoFile.create(f)) + await Promise.all(tasks) + + const tags = videoToCreateData.tag.map(t => t.name) + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + await videoCreated.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) +} diff --git a/server/lib/activitypub/process-create.ts b/server/lib/activitypub/process-create.ts index 114ff1848..471674ead 100644 --- a/server/lib/activitypub/process-create.ts +++ b/server/lib/activitypub/process-create.ts @@ -1,23 +1,23 @@ -import { - ActivityCreate, - VideoTorrentObject, - VideoChannelObject -} from '../../../shared' +import { ActivityCreate, VideoChannelObject, VideoTorrentObject } from '../../../shared' +import { ActivityAdd } from '../../../shared/models/activitypub/activity' +import { generateThumbnailFromUrl, logger, retryTransactionWrapper } from '../../helpers' import { database as db } from '../../initializers' -import { logger, retryTransactionWrapper } from '../../helpers' +import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import Bluebird = require('bluebird') +import { AccountInstance } from '../../models/account/account-interface' +import { getActivityPubUrl, getOrCreateAccount } from '../../helpers/activitypub' -function processCreateActivity (activity: ActivityCreate) { +async function processCreateActivity (activity: ActivityCreate) { const activityObject = activity.object const activityType = activityObject.type + const account = await getOrCreateAccount(activity.actor) - if (activityType === 'Video') { - return processCreateVideo(activityObject as VideoTorrentObject) - } else if (activityType === 'VideoChannel') { - return processCreateVideoChannel(activityObject as VideoChannelObject) + if (activityType === 'VideoChannel') { + return processCreateVideoChannel(account, activityObject as VideoChannelObject) } logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) - return Promise.resolve() + return Promise.resolve(undefined) } // --------------------------------------------------------------------------- @@ -28,77 +28,37 @@ export { // --------------------------------------------------------------------------- -function processCreateVideo (video: VideoTorrentObject) { +function processCreateVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) { const options = { - arguments: [ video ], - errorMessage: 'Cannot insert the remote video with many retries.' + arguments: [ account, videoChannelToCreateData ], + errorMessage: 'Cannot insert the remote video channel with many retries.' } - return retryTransactionWrapper(addRemoteVideo, options) + return retryTransactionWrapper(addRemoteVideoChannel, options) } -async function addRemoteVideo (videoToCreateData: VideoTorrentObject) { - logger.debug('Adding remote video %s.', videoToCreateData.url) +async function addRemoteVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) { + logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid) await db.sequelize.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) - if (videoFromDatabase) throw new Error('UUID already exists.') - - const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) - if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') - - const tags = videoToCreateData.tags - const tagInstances = await db.Tag.findOrCreateTags(tags, t) - - const videoData = { - name: videoToCreateData.name, - uuid: videoToCreateData.uuid, - category: videoToCreateData.category, - licence: videoToCreateData.licence, - language: videoToCreateData.language, - nsfw: videoToCreateData.nsfw, - description: videoToCreateData.truncatedDescription, - channelId: videoChannel.id, - duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt, - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: videoToCreateData.updatedAt, - views: videoToCreateData.views, - likes: videoToCreateData.likes, - dislikes: videoToCreateData.dislikes, + let videoChannel = await db.VideoChannel.loadByUUIDOrUrl(videoChannelToCreateData.uuid, videoChannelToCreateData.id, t) + if (videoChannel) throw new Error('Video channel with this URL/UUID already exists.') + + const videoChannelData = { + name: videoChannelToCreateData.name, + description: videoChannelToCreateData.content, + uuid: videoChannelToCreateData.uuid, + createdAt: videoChannelToCreateData.published, + updatedAt: videoChannelToCreateData.updated, remote: true, - privacy: videoToCreateData.privacy - } - - const video = db.Video.build(videoData) - await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) - const videoCreated = await video.save(sequelizeOptions) - - const tasks = [] - for (const fileData of videoToCreateData.files) { - const videoFileInstance = db.VideoFile.build({ - extname: fileData.extname, - infoHash: fileData.infoHash, - resolution: fileData.resolution, - size: fileData.size, - videoId: videoCreated.id - }) - - tasks.push(videoFileInstance.save(sequelizeOptions)) + accountId: account.id } - await Promise.all(tasks) + videoChannel = db.VideoChannel.build(videoChannelData) + videoChannel.url = getActivityPubUrl('videoChannel', videoChannel.uuid) - await videoCreated.setTags(tagInstances, sequelizeOptions) + await videoChannel.save({ transaction: t }) }) - logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) -} - -function processCreateVideoChannel (videoChannel: VideoChannelObject) { - + logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid) } diff --git a/server/lib/activitypub/process-update.ts b/server/lib/activitypub/process-update.ts index 187c7be7c..cd8a4b8e2 100644 --- a/server/lib/activitypub/process-update.ts +++ b/server/lib/activitypub/process-update.ts @@ -1,15 +1,25 @@ -import { - ActivityCreate, - VideoTorrentObject, - VideoChannelObject -} from '../../../shared' +import { VideoChannelObject, VideoTorrentObject } from '../../../shared' +import { ActivityUpdate } from '../../../shared/models/activitypub/activity' +import { getOrCreateAccount } from '../../helpers/activitypub' +import { retryTransactionWrapper } from '../../helpers/database-utils' +import { logger } from '../../helpers/logger' +import { resetSequelizeInstance } from '../../helpers/utils' +import { database as db } from '../../initializers' +import { AccountInstance } from '../../models/account/account-interface' +import { VideoInstance } from '../../models/video/video-interface' +import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import Bluebird = require('bluebird') + +async function processUpdateActivity (activity: ActivityUpdate) { + const account = await getOrCreateAccount(activity.actor) -function processUpdateActivity (activity: ActivityCreate) { if (activity.object.type === 'Video') { - return processUpdateVideo(activity.object) + return processUpdateVideo(account, activity.object) } else if (activity.object.type === 'VideoChannel') { - return processUpdateVideoChannel(activity.object) + return processUpdateVideoChannel(account, activity.object) } + + return undefined } // --------------------------------------------------------------------------- @@ -20,10 +30,107 @@ export { // --------------------------------------------------------------------------- -function processUpdateVideo (video: VideoTorrentObject) { +function processUpdateVideo (account: AccountInstance, video: VideoTorrentObject) { + const options = { + arguments: [ account, video ], + errorMessage: 'Cannot update the remote video with many retries' + } + return retryTransactionWrapper(updateRemoteVideo, options) } -function processUpdateVideoChannel (videoChannel: VideoChannelObject) { +async function updateRemoteVideo (account: AccountInstance, videoAttributesToUpdate: VideoTorrentObject) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) + let videoInstance: VideoInstance + let videoFieldsSave: object + + try { + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoInstance = await db.Video.loadByUrl(videoAttributesToUpdate.id, t) + if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.') + + if (videoInstance.VideoChannel.Account.id !== account.id) { + throw new Error('Account ' + account.url + ' does not own video channel ' + videoInstance.VideoChannel.url) + } + + const videoData = await videoActivityObjectToDBAttributes(videoInstance.VideoChannel, videoAttributesToUpdate, t) + videoInstance.set('name', videoData.name) + videoInstance.set('category', videoData.category) + videoInstance.set('licence', videoData.licence) + videoInstance.set('language', videoData.language) + videoInstance.set('nsfw', videoData.nsfw) + videoInstance.set('description', videoData.description) + videoInstance.set('duration', videoData.duration) + videoInstance.set('createdAt', videoData.createdAt) + videoInstance.set('updatedAt', videoData.updatedAt) + videoInstance.set('views', videoData.views) + // videoInstance.set('likes', videoData.likes) + // videoInstance.set('dislikes', videoData.dislikes) + // videoInstance.set('privacy', videoData.privacy) + + await videoInstance.save(sequelizeOptions) + + // Remove old video files + const videoFileDestroyTasks: Bluebird[] = [] + for (const videoFile of videoInstance.VideoFiles) { + videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) + } + await Promise.all(videoFileDestroyTasks) + + const videoFileAttributes = await videoFileActivityUrlToDBAttributes(videoInstance, videoAttributesToUpdate) + const tasks: Bluebird[] = videoFileAttributes.map(f => db.VideoFile.create(f)) + await Promise.all(tasks) + + const tags = videoAttributesToUpdate.tag.map(t => t.name) + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + await videoInstance.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid) + } catch (err) { + if (videoInstance !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(videoInstance, videoFieldsSave) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', err) + throw err + } +} + +async function processUpdateVideoChannel (account: AccountInstance, videoChannel: VideoChannelObject) { + const options = { + arguments: [ account, videoChannel ], + errorMessage: 'Cannot update the remote video channel with many retries.' + } + + await retryTransactionWrapper(updateRemoteVideoChannel, options) +} + +async function updateRemoteVideoChannel (account: AccountInstance, videoChannel: VideoChannelObject) { + logger.debug('Updating remote video channel "%s".', videoChannel.uuid) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoChannelInstance = await db.VideoChannel.loadByUrl(videoChannel.id) + if (!videoChannelInstance) throw new Error('Video ' + videoChannel.id + ' not found.') + + if (videoChannelInstance.Account.id !== account.id) { + throw new Error('Account ' + account.id + ' does not own video channel ' + videoChannelInstance.url) + } + + videoChannelInstance.set('name', videoChannel.name) + videoChannelInstance.set('description', videoChannel.content) + videoChannelInstance.set('createdAt', videoChannel.published) + videoChannelInstance.set('updatedAt', videoChannel.updated) + + await videoChannelInstance.save(sequelizeOptions) + }) + logger.info('Remote video channel with uuid %s updated', videoChannel.uuid) } diff --git a/server/middlewares/validators/activitypub/activity.ts b/server/middlewares/validators/activitypub/activity.ts new file mode 100644 index 000000000..78a6d1444 --- /dev/null +++ b/server/middlewares/validators/activitypub/activity.ts @@ -0,0 +1,21 @@ +import { body } from 'express-validator/check' +import * as express from 'express' + +import { logger, isRootActivityValid } from '../../../helpers' +import { checkErrors } from '../utils' + +const activityPubValidator = [ + body('data').custom(isRootActivityValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking activity pub parameters', { parameters: req.body }) + + checkErrors(req, res, next) + } +] + +// --------------------------------------------------------------------------- + +export { + activityPubValidator +} diff --git a/server/middlewares/validators/activitypub/videos.ts b/server/middlewares/validators/activitypub/videos.ts deleted file mode 100644 index 497320cc1..000000000 --- a/server/middlewares/validators/activitypub/videos.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { body } from 'express-validator/check' -import * as express from 'express' - -import { - logger, - isArray, - removeBadRequestVideos, - removeBadRequestVideosQadu, - removeBadRequestVideosEvents -} from '../../../helpers' -import { checkErrors } from '../utils' - -const remoteVideosValidator = [ - body('data').custom(isArray), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking remoteVideos parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - removeBadRequestVideos(req.body.data) - - return next() - }) - } -] - -const remoteQaduVideosValidator = [ - body('data').custom(isArray), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - removeBadRequestVideosQadu(req.body.data) - - return next() - }) - } -] - -const remoteEventsVideosValidator = [ - body('data').custom(isArray), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body }) - - checkErrors(req, res, () => { - removeBadRequestVideosEvents(req.body.data) - - return next() - }) - } -] - -// --------------------------------------------------------------------------- - -export { - remoteVideosValidator, - remoteQaduVideosValidator, - remoteEventsVideosValidator -} diff --git a/server/models/video/video-channel-interface.ts b/server/models/video/video-channel-interface.ts index 477f97cd4..55e772063 100644 --- a/server/models/video/video-channel-interface.ts +++ b/server/models/video/video-channel-interface.ts @@ -24,6 +24,8 @@ export namespace VideoChannelMethods { export type LoadByUUID = (uuid: string, t?: Sequelize.Transaction) => Promise export type LoadByHostAndUUID = (uuid: string, podHost: string, t?: Sequelize.Transaction) => Promise export type LoadAndPopulateAccountAndVideos = (id: number) => Promise + export type LoadByUrl = (uuid: string, t?: Sequelize.Transaction) => Promise + export type LoadByUUIDOrUrl = (uuid: string, url: string, t?: Sequelize.Transaction) => Promise } export interface VideoChannelClass { @@ -37,6 +39,8 @@ export interface VideoChannelClass { loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos + loadByUrl: VideoChannelMethods.LoadByUrl + loadByUUIDOrUrl: VideoChannelMethods.LoadByUUIDOrUrl } export interface VideoChannelAttributes { @@ -45,7 +49,7 @@ export interface VideoChannelAttributes { name: string description: string remote: boolean - url: string + url?: string Account?: AccountInstance Videos?: VideoInstance[] diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index c17828f3e..93a611fa0 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -25,6 +25,8 @@ let loadAndPopulateAccount: VideoChannelMethods.LoadAndPopulateAccount let loadByUUIDAndPopulateAccount: VideoChannelMethods.LoadByUUIDAndPopulateAccount let loadByHostAndUUID: VideoChannelMethods.LoadByHostAndUUID let loadAndPopulateAccountAndVideos: VideoChannelMethods.LoadAndPopulateAccountAndVideos +let loadByUrl: VideoChannelMethods.LoadByUrl +let loadByUUIDOrUrl: VideoChannelMethods.LoadByUUIDOrUrl export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) { VideoChannel = sequelize.define('VideoChannel', @@ -94,12 +96,14 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da loadByUUID, loadByHostAndUUID, loadAndPopulateAccountAndVideos, - countByAccount + countByAccount, + loadByUrl, + loadByUUIDOrUrl ] const instanceMethods = [ isOwned, toFormattedJSON, - toActivityPubObject, + toActivityPubObject ] addMethodsToModel(VideoChannel, classMethods, instanceMethods) @@ -254,6 +258,33 @@ loadByUUID = function (uuid: string, t?: Sequelize.Transaction) { return VideoChannel.findOne(query) } +loadByUrl = function (url: string, t?: Sequelize.Transaction) { + const query: Sequelize.FindOptions = { + where: { + url + } + } + + if (t !== undefined) query.transaction = t + + return VideoChannel.findOne(query) +} + +loadByUUIDOrUrl = function (uuid: string, url: string, t?: Sequelize.Transaction) { + const query: Sequelize.FindOptions = { + where: { + [Sequelize.Op.or]: [ + { uuid }, + { url } + ] + }, + } + + if (t !== undefined) query.transaction = t + + return VideoChannel.findOne(query) +} + loadByHostAndUUID = function (fromHost: string, uuid: string, t?: Sequelize.Transaction) { const query: Sequelize.FindOptions = { where: { diff --git a/server/models/video/video-interface.ts b/server/models/video/video-interface.ts index e62e25a82..a0ac43e1e 100644 --- a/server/models/video/video-interface.ts +++ b/server/models/video/video-interface.ts @@ -69,6 +69,7 @@ export namespace VideoMethods { export type LoadAndPopulateAccount = (id: number) => Bluebird export type LoadAndPopulateAccountAndPodAndTags = (id: number) => Bluebird export type LoadByUUIDAndPopulateAccountAndPodAndTags = (uuid: string) => Bluebird + export type LoadByUUIDOrURL = (uuid: string, url: string, t?: Sequelize.Transaction) => Bluebird export type RemoveThumbnail = (this: VideoInstance) => Promise export type RemovePreview = (this: VideoInstance) => Promise @@ -89,6 +90,7 @@ export interface VideoClass { loadByHostAndUUID: VideoMethods.LoadByHostAndUUID loadByUUID: VideoMethods.LoadByUUID loadByUrl: VideoMethods.LoadByUrl + loadByUUIDOrURL: VideoMethods.LoadByUUIDOrURL loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID loadByUUIDAndPopulateAccountAndPodAndTags: VideoMethods.LoadByUUIDAndPopulateAccountAndPodAndTags searchAndPopulateAccountAndPodAndTags: VideoMethods.SearchAndPopulateAccountAndPodAndTags @@ -109,7 +111,10 @@ export interface VideoAttributes { likes?: number dislikes?: number remote: boolean - url: string + url?: string + + createdAt?: Date + updatedAt?: Date parentId?: number channelId?: number @@ -120,9 +125,6 @@ export interface VideoAttributes { } export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.Instance { - createdAt: Date - updatedAt: Date - createPreview: VideoMethods.CreatePreview createThumbnail: VideoMethods.CreateThumbnail createTorrentAndSetInfoHash: VideoMethods.CreateTorrentAndSetInfoHash @@ -158,4 +160,3 @@ export interface VideoInstance extends VideoClass, VideoAttributes, Sequelize.In } export interface VideoModel extends VideoClass, Sequelize.Model {} - diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 94af1ece5..b5d333347 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -25,7 +25,8 @@ import { statPromise, generateImageFromVideoFile, transcode, - getVideoFileHeight + getVideoFileHeight, + getActivityPubUrl } from '../../helpers' import { CONFIG, @@ -88,7 +89,7 @@ let listOwnedAndPopulateAccountAndTags: VideoMethods.ListOwnedAndPopulateAccount let listOwnedByAccount: VideoMethods.ListOwnedByAccount let load: VideoMethods.Load let loadByUUID: VideoMethods.LoadByUUID -let loadByUrl: VideoMethods.LoadByUrl +let loadByUUIDOrURL: VideoMethods.LoadByUUIDOrURL let loadLocalVideoByUUID: VideoMethods.LoadLocalVideoByUUID let loadAndPopulateAccount: VideoMethods.LoadAndPopulateAccount let loadAndPopulateAccountAndPodAndTags: VideoMethods.LoadAndPopulateAccountAndPodAndTags @@ -277,6 +278,7 @@ export default function (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.Da loadAndPopulateAccount, loadAndPopulateAccountAndPodAndTags, loadByHostAndUUID, + loadByUUIDOrURL, loadByUUID, loadLocalVideoByUUID, loadByUUIDAndPopulateAccountAndPodAndTags, @@ -595,6 +597,7 @@ toActivityPubObject = function (this: VideoInstance) { const videoObject: VideoTorrentObject = { type: 'Video', + id: getActivityPubUrl('video', this.uuid), name: this.name, // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration duration: 'PT' + this.duration + 'S', @@ -731,6 +734,7 @@ getCategoryLabel = function (this: VideoInstance) { getLicenceLabel = function (this: VideoInstance) { let licenceLabel = VIDEO_LICENCES[this.licence] + // Maybe our pod is not up to date and there are new licences since our version if (!licenceLabel) licenceLabel = 'Unknown' @@ -946,6 +950,22 @@ loadByUUID = function (uuid: string, t?: Sequelize.Transaction) { return Video.findOne(query) } +loadByUUIDOrURL = function (uuid: string, url: string, t?: Sequelize.Transaction) { + const query: Sequelize.FindOptions = { + where: { + [Sequelize.Op.or]: [ + { uuid }, + { url } + ] + }, + include: [ Video['sequelize'].models.VideoFile ] + } + + if (t !== undefined) query.transaction = t + + return Video.findOne(query) +} + loadLocalVideoByUUID = function (uuid: string, t?: Sequelize.Transaction) { const query: Sequelize.FindOptions = { where: { diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts index 0274416b2..dc562c00a 100644 --- a/shared/models/activitypub/activity.ts +++ b/shared/models/activitypub/activity.ts @@ -7,7 +7,7 @@ import { ActivityPubSignature } from './activitypub-signature' export type Activity = ActivityCreate | ActivityUpdate | ActivityFlag // Flag -> report abuse -export type ActivityType = 'Create' | 'Update' | 'Flag' +export type ActivityType = 'Create' | 'Add' | 'Update' | 'Flag' export interface BaseActivity { '@context'?: any[] @@ -20,7 +20,12 @@ export interface BaseActivity { export interface ActivityCreate extends BaseActivity { type: 'Create' - object: VideoTorrentObject | VideoChannelObject + object: VideoChannelObject +} + +export interface ActivityAdd extends BaseActivity { + type: 'Add' + object: VideoTorrentObject } export interface ActivityUpdate extends BaseActivity { diff --git a/shared/models/activitypub/objects/video-channel-object.ts b/shared/models/activitypub/objects/video-channel-object.ts index d64b4aed8..72efe42b3 100644 --- a/shared/models/activitypub/objects/video-channel-object.ts +++ b/shared/models/activitypub/objects/video-channel-object.ts @@ -2,7 +2,10 @@ import { ActivityIdentifierObject } from './common-objects' export interface VideoChannelObject { type: 'VideoChannel' + id: string name: string content: string - uuid: ActivityIdentifierObject + uuid: string + published: Date + updated: Date } diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-torrent-object.ts index 00cc0a649..5685a43e0 100644 --- a/shared/models/activitypub/objects/video-torrent-object.ts +++ b/shared/models/activitypub/objects/video-torrent-object.ts @@ -7,6 +7,7 @@ import { export interface VideoTorrentObject { type: 'Video' + id: string name: string duration: string uuid: string -- 2.41.0