From 0d0e8dd0904b380b70e19ebcb4763d65601c4632 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Nov 2017 14:34:45 +0100 Subject: Continue activitypub --- server/lib/activitypub/misc.ts | 77 +++++++++++++++++++ server/lib/activitypub/process-add.ts | 72 ++++++++++++++++++ server/lib/activitypub/process-create.ts | 104 ++++++++----------------- server/lib/activitypub/process-update.ts | 127 ++++++++++++++++++++++++++++--- 4 files changed, 298 insertions(+), 82 deletions(-) create mode 100644 server/lib/activitypub/misc.ts create mode 100644 server/lib/activitypub/process-add.ts (limited to 'server/lib/activitypub') diff --git a/server/lib/activitypub/misc.ts b/server/lib/activitypub/misc.ts new file mode 100644 index 000000000..05e77ebc3 --- /dev/null +++ b/server/lib/activitypub/misc.ts @@ -0,0 +1,77 @@ +import * as magnetUtil from 'magnet-uri' +import * as Sequelize from 'sequelize' +import { VideoTorrentObject } from '../../../shared' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { database as db } from '../../initializers' +import { VIDEO_MIMETYPE_EXT } from '../../initializers/constants' +import { VideoChannelInstance } from '../../models/video/video-channel-interface' +import { VideoFileAttributes } from '../../models/video/video-file-interface' +import { VideoAttributes, VideoInstance } from '../../models/video/video-interface' + +async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelInstance, videoObject: VideoTorrentObject, t: Sequelize.Transaction) { + const videoFromDatabase = await db.Video.loadByUUIDOrURL(videoObject.uuid, videoObject.id, t) + if (videoFromDatabase) throw new Error('Video with this UUID/Url already exists.') + + const duration = videoObject.duration.replace(/[^\d]+/, '') + const videoData: VideoAttributes = { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category: parseInt(videoObject.category.identifier, 10), + licence: parseInt(videoObject.licence.identifier, 10), + language: parseInt(videoObject.language.identifier, 10), + nsfw: videoObject.nsfw, + description: videoObject.content, + channelId: videoChannel.id, + duration: parseInt(duration, 10), + createdAt: videoObject.published, + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: videoObject.updated, + views: videoObject.views, + likes: 0, + dislikes: 0, + // likes: videoToCreateData.likes, + // dislikes: videoToCreateData.dislikes, + remote: true, + privacy: 1 + // privacy: videoToCreateData.privacy + } + + return videoData +} + +function videoFileActivityUrlToDBAttributes (videoCreated: VideoInstance, videoObject: VideoTorrentObject) { + const fileUrls = videoObject.url + .filter(u => Object.keys(VIDEO_MIMETYPE_EXT).indexOf(u.mimeType) !== -1) + + const attributes: VideoFileAttributes[] = [] + for (const url of fileUrls) { + // Fetch associated magnet uri + const magnet = videoObject.url + .find(u => { + return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === url.width + }) + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + url.url) + + const parsed = magnetUtil.decode(magnet.url) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url) + + const attribute = { + extname: VIDEO_MIMETYPE_EXT[url.mimeType], + infoHash: parsed.infoHash, + resolution: url.width, + size: url.size, + videoId: videoCreated.id + } + attributes.push(attribute) + } + + return attributes +} + +// --------------------------------------------------------------------------- + +export { + videoFileActivityUrlToDBAttributes, + videoActivityObjectToDBAttributes +} diff --git a/server/lib/activitypub/process-add.ts b/server/lib/activitypub/process-add.ts new file mode 100644 index 000000000..40541aca3 --- /dev/null +++ b/server/lib/activitypub/process-add.ts @@ -0,0 +1,72 @@ +import { VideoTorrentObject } from '../../../shared' +import { ActivityAdd } from '../../../shared/models/activitypub/activity' +import { generateThumbnailFromUrl, logger, retryTransactionWrapper, getOrCreateAccount } from '../../helpers' +import { database as db } from '../../initializers' +import { AccountInstance } from '../../models/account/account-interface' +import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import Bluebird = require('bluebird') + +async function processAddActivity (activity: ActivityAdd) { + const activityObject = activity.object + const activityType = activityObject.type + const account = await getOrCreateAccount(activity.actor) + + if (activityType === 'Video') { + return processAddVideo(account, activity.id, activityObject as VideoTorrentObject) + } + + logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) + return Promise.resolve(undefined) +} + +// --------------------------------------------------------------------------- + +export { + processAddActivity +} + +// --------------------------------------------------------------------------- + +function processAddVideo (account: AccountInstance, videoChannelUrl: string, video: VideoTorrentObject) { + const options = { + arguments: [ account, videoChannelUrl ,video ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + return retryTransactionWrapper(addRemoteVideo, options) +} + +async function addRemoteVideo (account: AccountInstance, videoChannelUrl: string, videoToCreateData: VideoTorrentObject) { + logger.debug('Adding remote video %s.', videoToCreateData.url) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoChannel = await db.VideoChannel.loadByUrl(videoChannelUrl, t) + if (!videoChannel) throw new Error('Video channel not found.') + + if (videoChannel.Account.id !== account.id) throw new Error('Video channel is not owned by this account.') + + const videoData = await videoActivityObjectToDBAttributes(videoChannel, videoToCreateData, t) + const video = db.Video.build(videoData) + + // Don't block on request + generateThumbnailFromUrl(video, videoToCreateData.icon) + .catch(err => logger.warning('Cannot generate thumbnail of %s.', videoToCreateData.id, err)) + + const videoCreated = await video.save(sequelizeOptions) + + const videoFileAttributes = await videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData) + + const tasks: Bluebird[] = videoFileAttributes.map(f => db.VideoFile.create(f)) + await Promise.all(tasks) + + const tags = videoToCreateData.tag.map(t => t.name) + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + await videoCreated.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) +} diff --git a/server/lib/activitypub/process-create.ts b/server/lib/activitypub/process-create.ts index 114ff1848..471674ead 100644 --- a/server/lib/activitypub/process-create.ts +++ b/server/lib/activitypub/process-create.ts @@ -1,23 +1,23 @@ -import { - ActivityCreate, - VideoTorrentObject, - VideoChannelObject -} from '../../../shared' +import { ActivityCreate, VideoChannelObject, VideoTorrentObject } from '../../../shared' +import { ActivityAdd } from '../../../shared/models/activitypub/activity' +import { generateThumbnailFromUrl, logger, retryTransactionWrapper } from '../../helpers' import { database as db } from '../../initializers' -import { logger, retryTransactionWrapper } from '../../helpers' +import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import Bluebird = require('bluebird') +import { AccountInstance } from '../../models/account/account-interface' +import { getActivityPubUrl, getOrCreateAccount } from '../../helpers/activitypub' -function processCreateActivity (activity: ActivityCreate) { +async function processCreateActivity (activity: ActivityCreate) { const activityObject = activity.object const activityType = activityObject.type + const account = await getOrCreateAccount(activity.actor) - if (activityType === 'Video') { - return processCreateVideo(activityObject as VideoTorrentObject) - } else if (activityType === 'VideoChannel') { - return processCreateVideoChannel(activityObject as VideoChannelObject) + if (activityType === 'VideoChannel') { + return processCreateVideoChannel(account, activityObject as VideoChannelObject) } logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) - return Promise.resolve() + return Promise.resolve(undefined) } // --------------------------------------------------------------------------- @@ -28,77 +28,37 @@ export { // --------------------------------------------------------------------------- -function processCreateVideo (video: VideoTorrentObject) { +function processCreateVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) { const options = { - arguments: [ video ], - errorMessage: 'Cannot insert the remote video with many retries.' + arguments: [ account, videoChannelToCreateData ], + errorMessage: 'Cannot insert the remote video channel with many retries.' } - return retryTransactionWrapper(addRemoteVideo, options) + return retryTransactionWrapper(addRemoteVideoChannel, options) } -async function addRemoteVideo (videoToCreateData: VideoTorrentObject) { - logger.debug('Adding remote video %s.', videoToCreateData.url) +async function addRemoteVideoChannel (account: AccountInstance, videoChannelToCreateData: VideoChannelObject) { + logger.debug('Adding remote video channel "%s".', videoChannelToCreateData.uuid) await db.sequelize.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - const videoFromDatabase = await db.Video.loadByUUID(videoToCreateData.uuid) - if (videoFromDatabase) throw new Error('UUID already exists.') - - const videoChannel = await db.VideoChannel.loadByHostAndUUID(fromPod.host, videoToCreateData.channelUUID, t) - if (!videoChannel) throw new Error('Video channel ' + videoToCreateData.channelUUID + ' not found.') - - const tags = videoToCreateData.tags - const tagInstances = await db.Tag.findOrCreateTags(tags, t) - - const videoData = { - name: videoToCreateData.name, - uuid: videoToCreateData.uuid, - category: videoToCreateData.category, - licence: videoToCreateData.licence, - language: videoToCreateData.language, - nsfw: videoToCreateData.nsfw, - description: videoToCreateData.truncatedDescription, - channelId: videoChannel.id, - duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt, - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: videoToCreateData.updatedAt, - views: videoToCreateData.views, - likes: videoToCreateData.likes, - dislikes: videoToCreateData.dislikes, + let videoChannel = await db.VideoChannel.loadByUUIDOrUrl(videoChannelToCreateData.uuid, videoChannelToCreateData.id, t) + if (videoChannel) throw new Error('Video channel with this URL/UUID already exists.') + + const videoChannelData = { + name: videoChannelToCreateData.name, + description: videoChannelToCreateData.content, + uuid: videoChannelToCreateData.uuid, + createdAt: videoChannelToCreateData.published, + updatedAt: videoChannelToCreateData.updated, remote: true, - privacy: videoToCreateData.privacy - } - - const video = db.Video.build(videoData) - await db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData) - const videoCreated = await video.save(sequelizeOptions) - - const tasks = [] - for (const fileData of videoToCreateData.files) { - const videoFileInstance = db.VideoFile.build({ - extname: fileData.extname, - infoHash: fileData.infoHash, - resolution: fileData.resolution, - size: fileData.size, - videoId: videoCreated.id - }) - - tasks.push(videoFileInstance.save(sequelizeOptions)) + accountId: account.id } - await Promise.all(tasks) + videoChannel = db.VideoChannel.build(videoChannelData) + videoChannel.url = getActivityPubUrl('videoChannel', videoChannel.uuid) - await videoCreated.setTags(tagInstances, sequelizeOptions) + await videoChannel.save({ transaction: t }) }) - logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) -} - -function processCreateVideoChannel (videoChannel: VideoChannelObject) { - + logger.info('Remote video channel with uuid %s inserted.', videoChannelToCreateData.uuid) } diff --git a/server/lib/activitypub/process-update.ts b/server/lib/activitypub/process-update.ts index 187c7be7c..cd8a4b8e2 100644 --- a/server/lib/activitypub/process-update.ts +++ b/server/lib/activitypub/process-update.ts @@ -1,15 +1,25 @@ -import { - ActivityCreate, - VideoTorrentObject, - VideoChannelObject -} from '../../../shared' +import { VideoChannelObject, VideoTorrentObject } from '../../../shared' +import { ActivityUpdate } from '../../../shared/models/activitypub/activity' +import { getOrCreateAccount } from '../../helpers/activitypub' +import { retryTransactionWrapper } from '../../helpers/database-utils' +import { logger } from '../../helpers/logger' +import { resetSequelizeInstance } from '../../helpers/utils' +import { database as db } from '../../initializers' +import { AccountInstance } from '../../models/account/account-interface' +import { VideoInstance } from '../../models/video/video-interface' +import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import Bluebird = require('bluebird') + +async function processUpdateActivity (activity: ActivityUpdate) { + const account = await getOrCreateAccount(activity.actor) -function processUpdateActivity (activity: ActivityCreate) { if (activity.object.type === 'Video') { - return processUpdateVideo(activity.object) + return processUpdateVideo(account, activity.object) } else if (activity.object.type === 'VideoChannel') { - return processUpdateVideoChannel(activity.object) + return processUpdateVideoChannel(account, activity.object) } + + return undefined } // --------------------------------------------------------------------------- @@ -20,10 +30,107 @@ export { // --------------------------------------------------------------------------- -function processUpdateVideo (video: VideoTorrentObject) { +function processUpdateVideo (account: AccountInstance, video: VideoTorrentObject) { + const options = { + arguments: [ account, video ], + errorMessage: 'Cannot update the remote video with many retries' + } + return retryTransactionWrapper(updateRemoteVideo, options) } -function processUpdateVideoChannel (videoChannel: VideoChannelObject) { +async function updateRemoteVideo (account: AccountInstance, videoAttributesToUpdate: VideoTorrentObject) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) + let videoInstance: VideoInstance + let videoFieldsSave: object + + try { + await db.sequelize.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + const videoInstance = await db.Video.loadByUrl(videoAttributesToUpdate.id, t) + if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.') + + if (videoInstance.VideoChannel.Account.id !== account.id) { + throw new Error('Account ' + account.url + ' does not own video channel ' + videoInstance.VideoChannel.url) + } + + const videoData = await videoActivityObjectToDBAttributes(videoInstance.VideoChannel, videoAttributesToUpdate, t) + videoInstance.set('name', videoData.name) + videoInstance.set('category', videoData.category) + videoInstance.set('licence', videoData.licence) + videoInstance.set('language', videoData.language) + videoInstance.set('nsfw', videoData.nsfw) + videoInstance.set('description', videoData.description) + videoInstance.set('duration', videoData.duration) + videoInstance.set('createdAt', videoData.createdAt) + videoInstance.set('updatedAt', videoData.updatedAt) + videoInstance.set('views', videoData.views) + // videoInstance.set('likes', videoData.likes) + // videoInstance.set('dislikes', videoData.dislikes) + // videoInstance.set('privacy', videoData.privacy) + + await videoInstance.save(sequelizeOptions) + + // Remove old video files + const videoFileDestroyTasks: Bluebird[] = [] + for (const videoFile of videoInstance.VideoFiles) { + videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) + } + await Promise.all(videoFileDestroyTasks) + + const videoFileAttributes = await videoFileActivityUrlToDBAttributes(videoInstance, videoAttributesToUpdate) + const tasks: Bluebird[] = videoFileAttributes.map(f => db.VideoFile.create(f)) + await Promise.all(tasks) + + const tags = videoAttributesToUpdate.tag.map(t => t.name) + const tagInstances = await db.Tag.findOrCreateTags(tags, t) + await videoInstance.setTags(tagInstances, sequelizeOptions) + }) + + logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid) + } catch (err) { + if (videoInstance !== undefined && videoFieldsSave !== undefined) { + resetSequelizeInstance(videoInstance, videoFieldsSave) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', err) + throw err + } +} + +async function processUpdateVideoChannel (account: AccountInstance, videoChannel: VideoChannelObject) { + const options = { + arguments: [ account, videoChannel ], + errorMessage: 'Cannot update the remote video channel with many retries.' + } + + await retryTransactionWrapper(updateRemoteVideoChannel, options) +} + +async function updateRemoteVideoChannel (account: AccountInstance, videoChannel: VideoChannelObject) { + logger.debug('Updating remote video channel "%s".', videoChannel.uuid) + + await db.sequelize.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoChannelInstance = await db.VideoChannel.loadByUrl(videoChannel.id) + if (!videoChannelInstance) throw new Error('Video ' + videoChannel.id + ' not found.') + + if (videoChannelInstance.Account.id !== account.id) { + throw new Error('Account ' + account.id + ' does not own video channel ' + videoChannelInstance.url) + } + + videoChannelInstance.set('name', videoChannel.name) + videoChannelInstance.set('description', videoChannel.content) + videoChannelInstance.set('createdAt', videoChannel.published) + videoChannelInstance.set('updatedAt', videoChannel.updated) + + await videoChannelInstance.save(sequelizeOptions) + }) + logger.info('Remote video channel with uuid %s updated', videoChannel.uuid) } -- cgit v1.2.3