From 2ccaeeb341ffe8c2609039bf4c6d8835b4650316 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Jan 2018 17:18:12 +0100 Subject: Fetch remote AP objects --- server/lib/activitypub/index.ts | 5 + server/lib/activitypub/process/misc.ts | 194 --------------- server/lib/activitypub/process/process-announce.ts | 19 +- server/lib/activitypub/process/process-create.ts | 127 +++------- server/lib/activitypub/process/process-like.ts | 10 +- server/lib/activitypub/process/process-undo.ts | 16 +- server/lib/activitypub/process/process-update.ts | 10 +- server/lib/activitypub/video-comments.ts | 156 ++++++++++++ server/lib/activitypub/video-rates.ts | 52 ++++ server/lib/activitypub/videos.ts | 273 +++++++++++++++++---- 10 files changed, 491 insertions(+), 371 deletions(-) delete mode 100644 server/lib/activitypub/process/misc.ts create mode 100644 server/lib/activitypub/video-comments.ts create mode 100644 server/lib/activitypub/video-rates.ts (limited to 'server/lib') diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts index 94ed1edaa..0779d1e91 100644 --- a/server/lib/activitypub/index.ts +++ b/server/lib/activitypub/index.ts @@ -5,3 +5,8 @@ export * from './fetch' export * from './share' export * from './videos' export * from './url' +export { videoCommentActivityObjectToDBAttributes } from './video-comments' +export { addVideoComments } from './video-comments' +export { addVideoComment } from './video-comments' +export { sendVideoRateChangeToFollowers } from './video-rates' +export { sendVideoRateChangeToOrigin } from './video-rates' diff --git a/server/lib/activitypub/process/misc.ts b/server/lib/activitypub/process/misc.ts deleted file mode 100644 index 461619ea7..000000000 --- a/server/lib/activitypub/process/misc.ts +++ /dev/null @@ -1,194 +0,0 @@ -import * as magnetUtil from 'magnet-uri' -import { VideoTorrentObject } from '../../../../shared' -import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { isVideoFileInfoHashValid } from '../../../helpers/custom-validators/videos' -import { logger } from '../../../helpers/logger' -import { doRequest } from '../../../helpers/requests' -import { ACTIVITY_PUB, VIDEO_MIMETYPE_EXT } from '../../../initializers' -import { ActorModel } from '../../../models/activitypub/actor' -import { VideoModel } from '../../../models/video/video' -import { VideoChannelModel } from '../../../models/video/video-channel' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoShareModel } from '../../../models/video/video-share' -import { getOrCreateActorAndServerAndModel } from '../actor' - -async function videoActivityObjectToDBAttributes ( - videoChannel: VideoChannelModel, - videoObject: VideoTorrentObject, - to: string[] = [], - cc: string[] = [] -) { - let privacy = VideoPrivacy.PRIVATE - if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC - else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED - - const duration = videoObject.duration.replace(/[^\d]+/, '') - let language = null - if (videoObject.language) { - language = parseInt(videoObject.language.identifier, 10) - } - - let category = null - if (videoObject.category) { - category = parseInt(videoObject.category.identifier, 10) - } - - let licence = null - if (videoObject.licence) { - licence = parseInt(videoObject.licence.identifier, 10) - } - - let description = null - if (videoObject.content) { - description = videoObject.content - } - - return { - name: videoObject.name, - uuid: videoObject.uuid, - url: videoObject.id, - category, - licence, - language, - description, - nsfw: videoObject.nsfw, - commentsEnabled: videoObject.commentsEnabled, - channelId: videoChannel.id, - duration: parseInt(duration, 10), - createdAt: new Date(videoObject.published), - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: new Date(videoObject.updated), - views: videoObject.views, - likes: 0, - dislikes: 0, - remote: true, - privacy - } -} - -function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { - const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) - const fileUrls = videoObject.url.filter(u => { - return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/') - }) - - if (fileUrls.length === 0) { - throw new Error('Cannot find video files for ' + videoCreated.url) - } - - const attributes = [] - for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = videoObject.url.find(u => { - return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width - }) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url) - - const parsed = magnetUtil.decode(magnet.url) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url) - - const attribute = { - extname: VIDEO_MIMETYPE_EXT[fileUrl.mimeType], - infoHash: parsed.infoHash, - resolution: fileUrl.width, - size: fileUrl.size, - videoId: videoCreated.id - } - attributes.push(attribute) - } - - return attributes -} - -async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { - let originCommentId: number = null - let inReplyToCommentId: number = null - - // If this is not a reply to the video (thread), create or get the parent comment - if (video.url !== comment.inReplyTo) { - const [ parent ] = await addVideoComment(video, comment.inReplyTo) - if (!parent) { - logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id) - return undefined - } - - originCommentId = parent.originCommentId || parent.id - inReplyToCommentId = parent.id - } - - return { - url: comment.url, - text: comment.content, - videoId: video.id, - accountId: actor.Account.id, - inReplyToCommentId, - originCommentId, - createdAt: new Date(comment.published), - updatedAt: new Date(comment.updated) - } -} - -async function addVideoShares (instance: VideoModel, shareUrls: string[]) { - for (const shareUrl of shareUrls) { - // Fetch url - const { body } = await doRequest({ - uri: shareUrl, - json: true, - activityPub: true - }) - const actorUrl = body.actor - if (!actorUrl) continue - - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - - const entry = { - actorId: actor.id, - videoId: instance.id - } - - await VideoShareModel.findOrCreate({ - where: entry, - defaults: entry - }) - } -} - -async function addVideoComments (instance: VideoModel, commentUrls: string[]) { - for (const commentUrl of commentUrls) { - await addVideoComment(instance, commentUrl) - } -} - -async function addVideoComment (instance: VideoModel, commentUrl: string) { - // Fetch url - const { body } = await doRequest({ - uri: commentUrl, - json: true, - activityPub: true - }) - - const actorUrl = body.attributedTo - if (!actorUrl) return [] - - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - const entry = await videoCommentActivityObjectToDBAttributes(instance, actor, body) - if (!entry) return [] - - return VideoCommentModel.findOrCreate({ - where: { - url: body.id - }, - defaults: entry - }) -} - -// --------------------------------------------------------------------------- - -export { - videoFileActivityUrlToDBAttributes, - videoActivityObjectToDBAttributes, - addVideoShares, - addVideoComments -} diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 9adb40e01..bf7d7879d 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -7,6 +7,7 @@ import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity } from '../send/misc' +import { getOrCreateAccountAndVideoAndChannel } from '../videos' import { processCreateActivity } from './process-create' async function processAnnounceActivity (activity: ActivityAnnounce) { @@ -44,19 +45,19 @@ function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnoun return retryTransactionWrapper(shareVideo, options) } -function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { +async function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const announced = activity.object + let video: VideoModel + + if (typeof announced === 'string') { + const res = await getOrCreateAccountAndVideoAndChannel(announced) + video = res.video + } else { + video = await processCreateActivity(announced) + } return sequelizeTypescript.transaction(async t => { // Add share entry - let video: VideoModel - - if (typeof announced === 'string') { - video = await VideoModel.loadByUrlAndPopulateAccount(announced) - if (!video) throw new Error('Unknown video to share ' + announced) - } else { - video = await processCreateActivity(announced) - } const share = { actorId: actorAnnouncer.id, diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index e65b257c0..08d61996a 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -8,15 +8,13 @@ import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { TagModel } from '../../../models/video/tag' import { VideoModel } from '../../../models/video/video' import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoFileModel } from '../../../models/video/video-file' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity, getActorsInvolvedInVideo } from '../send/misc' -import { generateThumbnailFromUrl } from '../videos' -import { addVideoComments, addVideoShares, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import { addVideoComments, resolveThread } from '../video-comments' +import { addVideoShares, getOrCreateAccountAndVideoAndChannel } from '../videos' async function processCreateActivity (activity: ActivityCreate) { const activityObject = activity.object @@ -53,17 +51,7 @@ async function processCreateVideo ( ) { const videoToCreateData = activity.object as VideoTorrentObject - const channel = videoToCreateData.attributedTo.find(a => a.type === 'Group') - if (!channel) throw new Error('Cannot find associated video channel to video ' + videoToCreateData.url) - - const channelActor = await getOrCreateActorAndServerAndModel(channel.id) - - const options = { - arguments: [ actor, activity, videoToCreateData, channelActor ], - errorMessage: 'Cannot insert the remote video with many retries.' - } - - const video = await retryTransactionWrapper(createRemoteVideo, options) + const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData, actor) // Process outside the transaction because we could fetch remote data if (videoToCreateData.likes && Array.isArray(videoToCreateData.likes.orderedItems)) { @@ -89,48 +77,6 @@ async function processCreateVideo ( return video } -function createRemoteVideo ( - account: ActorModel, - activity: ActivityCreate, - videoToCreateData: VideoTorrentObject, - channelActor: ActorModel -) { - logger.debug('Adding remote video %s.', videoToCreateData.id) - - return sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - const videoFromDatabase = await VideoModel.loadByUUIDOrURL(videoToCreateData.uuid, videoToCreateData.id, t) - if (videoFromDatabase) return videoFromDatabase - - const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoToCreateData, activity.to, activity.cc) - const video = VideoModel.build(videoData) - - // Don't block on request - generateThumbnailFromUrl(video, videoToCreateData.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoToCreateData.id, err)) - - const videoCreated = await video.save(sequelizeOptions) - - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData) - if (videoFileAttributes.length === 0) { - throw new Error('Cannot find valid files for video %s ' + videoToCreateData.url) - } - - const tasks: Bluebird[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) - await Promise.all(tasks) - - const tags = videoToCreateData.tag.map(t => t.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await videoCreated.$set('Tags', tagInstances, sequelizeOptions) - - logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) - - return videoCreated - }) -} - async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { let rateCounts = 0 const tasks: Bluebird[] = [] @@ -167,16 +113,15 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea return retryTransactionWrapper(createVideoDislike, options) } -function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { +async function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { const dislike = activity.object as DislikeObject const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t) - if (!video) throw new Error('Unknown video ' + dislike.object) + const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + return sequelizeTypescript.transaction(async t => { const rate = { type: 'dislike' as 'dislike', videoId: video.id, @@ -200,9 +145,7 @@ function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const video = await VideoModel.loadByUrlAndPopulateAccount(view.object) - - if (!video) throw new Error('Unknown video ' + view.object) + const { video } = await getOrCreateAccountAndVideoAndChannel(view.object) const account = await ActorModel.loadByUrl(view.actor) if (!account) throw new Error('Unknown account ' + view.actor) @@ -225,19 +168,15 @@ function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: Vid return retryTransactionWrapper(addRemoteVideoAbuse, options) } -function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { +async function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) const account = actor.Account if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(videoAbuseToCreateData.object, t) - if (!video) { - logger.warn('Unknown video %s for remote video abuse.', videoAbuseToCreateData.object) - return undefined - } + const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object) + return sequelizeTypescript.transaction(async t => { const videoAbuseData = { reporterAccountId: account.id, reason: videoAbuseToCreateData.content, @@ -259,41 +198,33 @@ function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreat return retryTransactionWrapper(createVideoComment, options) } -function createVideoComment (byActor: ActorModel, activity: ActivityCreate) { +async function createVideoComment (byActor: ActorModel, activity: ActivityCreate) { const comment = activity.object as VideoCommentObject const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) + const { video, parents } = await resolveThread(comment.inReplyTo) + return sequelizeTypescript.transaction(async t => { - let video = await VideoModel.loadByUrlAndPopulateAccount(comment.inReplyTo, t) - let objectToCreate + let originCommentId = null + let inReplyToCommentId = null + + if (parents.length !== 0) { + const parent = parents[0] + + originCommentId = parent.getThreadId() + inReplyToCommentId = parent.id + } // This is a new thread - if (video) { - objectToCreate = { - url: comment.id, - text: comment.content, - originCommentId: null, - inReplyToComment: null, - videoId: video.id, - accountId: byAccount.id - } - } else { - const inReplyToComment = await VideoCommentModel.loadByUrl(comment.inReplyTo, t) - if (!inReplyToComment) throw new Error('Unknown replied comment ' + comment.inReplyTo) - - video = await VideoModel.loadAndPopulateAccount(inReplyToComment.videoId) - - const originCommentId = inReplyToComment.originCommentId || inReplyToComment.id - objectToCreate = { - url: comment.id, - text: comment.content, - originCommentId, - inReplyToCommentId: inReplyToComment.id, - videoId: video.id, - accountId: byAccount.id - } + const objectToCreate = { + url: comment.id, + text: comment.content, + originCommentId, + inReplyToCommentId, + videoId: video.id, + accountId: byAccount.id } const options = { diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 77fadabe1..0d161b126 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -3,9 +3,9 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { VideoModel } from '../../../models/video/video' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity } from '../send/misc' +import { getOrCreateAccountAndVideoAndChannel } from '../videos' async function processLikeActivity (activity: ActivityLike) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -30,17 +30,15 @@ async function processLikeVideo (actor: ActorModel, activity: ActivityLike) { return retryTransactionWrapper(createVideoLike, options) } -function createVideoLike (byActor: ActorModel, activity: ActivityLike) { +async function createVideoLike (byActor: ActorModel, activity: ActivityLike) { const videoUrl = activity.object const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) - - if (!video) throw new Error('Unknown video ' + videoUrl) + const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl) + return sequelizeTypescript.transaction(async t => { const rate = { type: 'like' as 'like', videoId: video.id, diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 9cad59233..5a770bb97 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -7,8 +7,8 @@ import { AccountModel } from '../../../models/account/account' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { VideoModel } from '../../../models/video/video' import { forwardActivity } from '../send/misc' +import { getOrCreateAccountAndVideoAndChannel } from '../videos' async function processUndoActivity (activity: ActivityUndo) { const activityToUndo = activity.object @@ -43,16 +43,15 @@ function processUndoLike (actorUrl: string, activity: ActivityUndo) { return retryTransactionWrapper(undoLike, options) } -function undoLike (actorUrl: string, activity: ActivityUndo) { +async function undoLike (actorUrl: string, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike + const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object) + return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) if (!byAccount) throw new Error('Unknown account ' + actorUrl) - const video = await VideoModel.loadByUrlAndPopulateAccount(likeActivity.object, t) - if (!video) throw new Error('Unknown video ' + likeActivity.actor) - const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) @@ -76,16 +75,15 @@ function processUndoDislike (actorUrl: string, activity: ActivityUndo) { return retryTransactionWrapper(undoDislike, options) } -function undoDislike (actorUrl: string, activity: ActivityUndo) { +async function undoDislike (actorUrl: string, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject + const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) if (!byAccount) throw new Error('Unknown account ' + actorUrl) - const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t) - if (!video) throw new Error('Unknown video ' + dislike.actor) - const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 2c094f7ca..a5431c76b 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -9,10 +9,9 @@ import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' import { TagModel } from '../../../models/video/tag' -import { VideoModel } from '../../../models/video/video' import { VideoFileModel } from '../../../models/video/video-file' import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' -import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import { getOrCreateAccountAndVideoAndChannel, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from '../videos' async function processUpdateActivity (activity: ActivityUpdate) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -46,8 +45,10 @@ function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) { async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) { const videoAttributesToUpdate = activity.object as VideoTorrentObject + const res = await getOrCreateAccountAndVideoAndChannel(videoAttributesToUpdate.id) + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) - let videoInstance: VideoModel + let videoInstance = res.video let videoFieldsSave: any try { @@ -56,9 +57,6 @@ async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) { transaction: t } - const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(videoAttributesToUpdate.id, t) - if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.') - videoFieldsSave = videoInstance.toJSON() const videoChannel = videoInstance.VideoChannel diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts new file mode 100644 index 000000000..17c86a381 --- /dev/null +++ b/server/lib/activitypub/video-comments.ts @@ -0,0 +1,156 @@ +import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object' +import { isVideoCommentObjectValid } from '../../helpers/custom-validators/activitypub/video-comments' +import { logger } from '../../helpers/logger' +import { doRequest } from '../../helpers/requests' +import { ACTIVITY_PUB } from '../../initializers' +import { ActorModel } from '../../models/activitypub/actor' +import { VideoModel } from '../../models/video/video' +import { VideoCommentModel } from '../../models/video/video-comment' +import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateAccountAndVideoAndChannel } from './videos' + +async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { + let originCommentId: number = null + let inReplyToCommentId: number = null + + // If this is not a reply to the video (thread), create or get the parent comment + if (video.url !== comment.inReplyTo) { + const [ parent ] = await addVideoComment(video, comment.inReplyTo) + if (!parent) { + logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id) + return undefined + } + + originCommentId = parent.originCommentId || parent.id + inReplyToCommentId = parent.id + } + + return { + url: comment.url, + text: comment.content, + videoId: video.id, + accountId: actor.Account.id, + inReplyToCommentId, + originCommentId, + createdAt: new Date(comment.published), + updatedAt: new Date(comment.updated) + } +} + +async function addVideoComments (instance: VideoModel, commentUrls: string[]) { + for (const commentUrl of commentUrls) { + await addVideoComment(instance, commentUrl) + } +} + +async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { + logger.info('Fetching remote video comment %s.', commentUrl) + + const { body } = await doRequest({ + uri: commentUrl, + json: true, + activityPub: true + }) + + if (isVideoCommentObjectValid(body) === false) { + logger.debug('Remote video comment JSON is not valid.', { body }) + return undefined + } + + const actorUrl = body.attributedTo + if (!actorUrl) return [] + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) + if (!entry) return [] + + return VideoCommentModel.findOrCreate({ + where: { + url: body.id + }, + defaults: entry + }) +} + +async function resolveThread (url: string, comments: VideoCommentModel[] = []) { + // Already have this comment? + const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideo(url) + if (commentFromDatabase) { + let parentComments = comments.concat([ commentFromDatabase ]) + + // Speed up things and resolve directly the thread + if (commentFromDatabase.InReplyToVideoComment) { + const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') + console.log(data) + + parentComments = parentComments.concat(data) + } + + return resolveThread(commentFromDatabase.Video.url, parentComments) + } + + try { + // Maybe it's a reply to a video? + const { video } = await getOrCreateAccountAndVideoAndChannel(url) + + if (comments.length !== 0) { + const firstReply = comments[ comments.length - 1 ] + firstReply.inReplyToCommentId = null + firstReply.originCommentId = null + firstReply.videoId = video.id + comments[comments.length - 1] = await firstReply.save() + + for (let i = comments.length - 2; i >= 0; i--) { + const comment = comments[ i ] + comment.originCommentId = firstReply.id + comment.inReplyToCommentId = comments[ i + 1 ].id + comment.videoId = video.id + + comments[i] = await comment.save() + } + } + + return { video, parents: comments } + } catch (err) { + logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, err) + + if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { + throw new Error('Recursion limit reached when resolving a thread') + } + + const { body } = await doRequest({ + uri: url, + json: true, + activityPub: true + }) + + if (isVideoCommentObjectValid(body) === false) { + throw new Error('Remote video comment JSON is not valid :' + JSON.stringify(body)) + } + + const actorUrl = body.attributedTo + if (!actorUrl) throw new Error('Miss attributed to in comment') + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const comment = new VideoCommentModel({ + url: body.url, + text: body.content, + videoId: null, + accountId: actor.Account.id, + inReplyToCommentId: null, + originCommentId: null, + createdAt: new Date(body.published), + updatedAt: new Date(body.updated) + }) + + return resolveThread(body.inReplyTo, comments.concat([ comment ])) + } + +} + +export { + videoCommentActivityObjectToDBAttributes, + addVideoComments, + addVideoComment, + resolveThread +} diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts new file mode 100644 index 000000000..1b2958cca --- /dev/null +++ b/server/lib/activitypub/video-rates.ts @@ -0,0 +1,52 @@ +import { Transaction } from 'sequelize' +import { AccountModel } from '../../models/account/account' +import { VideoModel } from '../../models/video/video' +import { + sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin, + sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers +} from './send' + +async function sendVideoRateChangeToFollowers (account: AccountModel, + video: VideoModel, + likes: number, + dislikes: number, + t: Transaction) { + const actor = account.Actor + + // Keep the order: first we undo and then we create + + // Undo Like + if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t) + // Undo Dislike + if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t) + + // Like + if (likes > 0) await sendLikeToVideoFollowers(actor, video, t) + // Dislike + if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t) +} + +async function sendVideoRateChangeToOrigin (account: AccountModel, + video: VideoModel, + likes: number, + dislikes: number, + t: Transaction) { + const actor = account.Actor + + // Keep the order: first we undo and then we create + + // Undo Like + if (likes < 0) await sendUndoLikeToOrigin(actor, video, t) + // Undo Dislike + if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t) + + // Like + if (likes > 0) await sendLikeToOrigin(actor, video, t) + // Dislike + if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t) +} + +export { + sendVideoRateChangeToFollowers, + sendVideoRateChangeToOrigin +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 8bc928b93..708f4a897 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -1,15 +1,23 @@ +import * as Bluebird from 'bluebird' +import * as magnetUtil from 'magnet-uri' import { join } from 'path' import * as request from 'request' -import { Transaction } from 'sequelize' import { ActivityIconObject } from '../../../shared/index' +import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' +import { VideoPrivacy } from '../../../shared/models/videos' +import { isVideoTorrentObjectValid } from '../../helpers/custom-validators/activitypub/videos' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { retryTransactionWrapper } from '../../helpers/database-utils' +import { logger } from '../../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { CONFIG, REMOTE_SCHEME, STATIC_PATHS } from '../../initializers' -import { AccountModel } from '../../models/account/account' +import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, STATIC_PATHS, VIDEO_MIMETYPE_EXT } from '../../initializers' +import { ActorModel } from '../../models/activitypub/actor' +import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' -import { - sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin, - sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers -} from './send' +import { VideoChannelModel } from '../../models/video/video-channel' +import { VideoFileModel } from '../../models/video/video-file' +import { VideoShareModel } from '../../models/video/video-share' +import { getOrCreateActorAndServerAndModel } from './actor' function fetchRemoteVideoPreview (video: VideoModel, reject: Function) { // FIXME: use url @@ -45,54 +53,221 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) return doRequestAndSaveToFile(options, thumbnailPath) } -async function sendVideoRateChangeToFollowers ( - account: AccountModel, - video: VideoModel, - likes: number, - dislikes: number, - t: Transaction -) { - const actor = account.Actor - - // Keep the order: first we undo and then we create - - // Undo Like - if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t) - // Undo Dislike - if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t) - - // Like - if (likes > 0) await sendLikeToVideoFollowers(actor, video, t) - // Dislike - if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t) +async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelModel, + videoObject: VideoTorrentObject, + to: string[] = [], + cc: string[] = []) { + let privacy = VideoPrivacy.PRIVATE + if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC + else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED + + const duration = videoObject.duration.replace(/[^\d]+/, '') + let language = null + if (videoObject.language) { + language = parseInt(videoObject.language.identifier, 10) + } + + let category = null + if (videoObject.category) { + category = parseInt(videoObject.category.identifier, 10) + } + + let licence = null + if (videoObject.licence) { + licence = parseInt(videoObject.licence.identifier, 10) + } + + let description = null + if (videoObject.content) { + description = videoObject.content + } + + return { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category, + licence, + language, + description, + nsfw: videoObject.nsfw, + commentsEnabled: videoObject.commentsEnabled, + channelId: videoChannel.id, + duration: parseInt(duration, 10), + createdAt: new Date(videoObject.published), + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: new Date(videoObject.updated), + views: videoObject.views, + likes: 0, + dislikes: 0, + remote: true, + privacy + } +} + +function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { + const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) + const fileUrls = videoObject.url.filter(u => { + return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/') + }) + + if (fileUrls.length === 0) { + throw new Error('Cannot find video files for ' + videoCreated.url) + } + + const attributes = [] + for (const fileUrl of fileUrls) { + // Fetch associated magnet uri + const magnet = videoObject.url.find(u => { + return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width + }) + + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url) + + const parsed = magnetUtil.decode(magnet.url) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url) + + const attribute = { + extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], + infoHash: parsed.infoHash, + resolution: fileUrl.width, + size: fileUrl.size, + videoId: videoCreated.id + } + attributes.push(attribute) + } + + return attributes +} + +async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) { + logger.debug('Adding remote video %s.', videoObject.id) + + return sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t) + if (videoFromDatabase) return videoFromDatabase + + const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to, videoObject.cc) + const video = VideoModel.build(videoData) + + // Don't block on request + generateThumbnailFromUrl(video, videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, err)) + + const videoCreated = await video.save(sequelizeOptions) + + const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) + if (videoFileAttributes.length === 0) { + throw new Error('Cannot find valid files for video %s ' + videoObject.url) + } + + const tasks: Bluebird[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) + await Promise.all(tasks) + + const tags = videoObject.tag.map(t => t.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await videoCreated.$set('Tags', tagInstances, sequelizeOptions) + + logger.info('Remote video with uuid %s inserted.', videoObject.uuid) + + videoCreated.VideoChannel = channelActor.VideoChannel + return videoCreated + }) } -async function sendVideoRateChangeToOrigin ( - account: AccountModel, - video: VideoModel, - likes: number, - dislikes: number, - t: Transaction -) { - const actor = account.Actor - - // Keep the order: first we undo and then we create - - // Undo Like - if (likes < 0) await sendUndoLikeToOrigin(actor, video, t) - // Undo Dislike - if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t) - - // Like - if (likes > 0) await sendLikeToOrigin(actor, video, t) - // Dislike - if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t) +async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) { + if (typeof videoObject === 'string') { + const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoObject) + if (videoFromDatabase) { + return { + video: videoFromDatabase, + actor: videoFromDatabase.VideoChannel.Account.Actor, + channelActor: videoFromDatabase.VideoChannel.Actor + } + } + + videoObject = await fetchRemoteVideo(videoObject) + if (!videoObject) throw new Error('Cannot fetch remote video') + } + + if (!actor) { + const actorObj = videoObject.attributedTo.find(a => a.type === 'Person') + if (!actorObj) throw new Error('Cannot find associated actor to video ' + videoObject.url) + + actor = await getOrCreateActorAndServerAndModel(actorObj.id) + } + + const channel = videoObject.attributedTo.find(a => a.type === 'Group') + if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) + + const channelActor = await getOrCreateActorAndServerAndModel(channel.id) + + const options = { + arguments: [ videoObject, channelActor ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + const video = await retryTransactionWrapper(getOrCreateVideo, options) + + return { actor, channelActor, video } +} + +async function addVideoShares (instance: VideoModel, shareUrls: string[]) { + for (const shareUrl of shareUrls) { + // Fetch url + const { body } = await doRequest({ + uri: shareUrl, + json: true, + activityPub: true + }) + const actorUrl = body.actor + if (!actorUrl) continue + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + + const entry = { + actorId: actor.id, + videoId: instance.id + } + + await VideoShareModel.findOrCreate({ + where: entry, + defaults: entry + }) + } } export { + getOrCreateAccountAndVideoAndChannel, fetchRemoteVideoPreview, fetchRemoteVideoDescription, generateThumbnailFromUrl, - sendVideoRateChangeToFollowers, - sendVideoRateChangeToOrigin + videoActivityObjectToDBAttributes, + videoFileActivityUrlToDBAttributes, + getOrCreateVideo, + addVideoShares} + +// --------------------------------------------------------------------------- + +async function fetchRemoteVideo (videoUrl: string): Promise { + const options = { + uri: videoUrl, + method: 'GET', + json: true, + activityPub: true + } + + logger.info('Fetching remote video %s.', videoUrl) + + const { body } = await doRequest(options) + + if (isVideoTorrentObjectValid(body) === false) { + logger.debug('Remote video JSON is not valid.', { body }) + return undefined + } + + return body } -- cgit v1.2.3