From 2ccaeeb341ffe8c2609039bf4c6d8835b4650316 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Jan 2018 17:18:12 +0100 Subject: [PATCH] Fetch remote AP objects --- client/src/standalone/videos/embed.ts | 1 + server/controllers/activitypub/client.ts | 6 +- .../activitypub/video-comments.ts | 3 +- .../custom-validators/activitypub/videos.ts | 3 +- server/initializers/constants.ts | 1 + server/initializers/installer.ts | 2 +- server/initializers/migrator.ts | 4 +- server/lib/activitypub/index.ts | 5 + server/lib/activitypub/process/misc.ts | 194 ------------- .../activitypub/process/process-announce.ts | 19 +- .../lib/activitypub/process/process-create.ts | 127 ++------ .../lib/activitypub/process/process-like.ts | 10 +- .../lib/activitypub/process/process-undo.ts | 16 +- .../lib/activitypub/process/process-update.ts | 10 +- server/lib/activitypub/video-comments.ts | 156 ++++++++++ server/lib/activitypub/video-rates.ts | 52 ++++ server/lib/activitypub/videos.ts | 273 ++++++++++++++---- server/models/activitypub/actor.ts | 3 + server/models/video/video-comment.ts | 15 +- server/models/video/video.ts | 12 +- server/tests/api/server/handle-down.ts | 147 ++++++++++ .../objects/video-torrent-object.ts | 2 + 22 files changed, 676 insertions(+), 385 deletions(-) delete mode 100644 server/lib/activitypub/process/misc.ts create mode 100644 server/lib/activitypub/video-comments.ts create mode 100644 server/lib/activitypub/video-rates.ts create mode 100644 server/tests/api/server/handle-down.ts diff --git a/client/src/standalone/videos/embed.ts b/client/src/standalone/videos/embed.ts index e68ee193a..9c672529f 100644 --- a/client/src/standalone/videos/embed.ts +++ b/client/src/standalone/videos/embed.ts @@ -1,6 +1,7 @@ import './embed.scss' import * as videojs from 'video.js' +import 'videojs-hotkeys' import '../../assets/player/peertube-videojs-plugin' import 'videojs-dock/dist/videojs-dock.es.js' import { VideoDetails } from '../../../../shared' diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index d1a761724..ec3f72b64 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -1,9 +1,11 @@ // Intercept ActivityPub client requests import * as express from 'express' +import { VideoPrivacy } from '../../../shared/models/videos' import { activityPubCollectionPagination } from '../../helpers/activitypub' import { pageToStartAndCount } from '../../helpers/core-utils' import { ACTIVITY_PUB, CONFIG } from '../../initializers' import { buildVideoAnnounceToFollowers } from '../../lib/activitypub/send' +import { audiencify, getAudience } from '../../lib/activitypub/send/misc' import { asyncMiddleware, executeIfActivityPub, localAccountValidator } from '../../middlewares' import { videoChannelsGetValidator, videosGetValidator, videosShareValidator } from '../../middlewares/validators' import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' @@ -95,7 +97,9 @@ async function videoController (req: express.Request, res: express.Response, nex // We need more attributes const videoAll = await VideoModel.loadAndPopulateAll(video.id) - return res.json(videoAll.toActivityPubObject()) + const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC) + + return res.json(audiencify(videoAll.toActivityPubObject(), audience)) } async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) { diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts index ce1209035..cbd4dac5c 100644 --- a/server/helpers/custom-validators/activitypub/video-comments.ts +++ b/server/helpers/custom-validators/activitypub/video-comments.ts @@ -24,7 +24,8 @@ function isVideoCommentDeleteActivityValid (activity: any) { export { isVideoCommentCreateActivityValid, - isVideoCommentDeleteActivityValid + isVideoCommentDeleteActivityValid, + isVideoCommentObjectValid } // --------------------------------------------------------------------------- diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 37cd6965a..fb1d2d094 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -70,7 +70,8 @@ export { isVideoTorrentCreateActivityValid, isVideoTorrentUpdateActivityValid, isVideoTorrentDeleteActivityValid, - isVideoFlagValid + isVideoFlagValid, + isVideoTorrentObjectValid } // --------------------------------------------------------------------------- diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c902e0cf6..c735e6daf 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -279,6 +279,7 @@ const ACTIVITY_PUB = { TORRENT: [ 'application/x-bittorrent' ], MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] }, + MAX_RECURSION_COMMENTS: 100, ACTOR_REFRESH_INTERVAL: 3600 * 24 // 1 day } diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index 58713c2c4..324a2c2e5 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -20,7 +20,7 @@ async function installApplication () { await createOAuthAdminIfNotExist() } catch (err) { logger.error('Cannot install application.', err) - throw err + process.exit(-1) } } diff --git a/server/initializers/migrator.ts b/server/initializers/migrator.ts index 29310b913..9ebc57f07 100644 --- a/server/initializers/migrator.ts +++ b/server/initializers/migrator.ts @@ -44,7 +44,7 @@ async function migrate () { await executeMigration(actualVersion, migrationScript) } catch (err) { logger.error('Cannot execute migration %s.', migrationScript.version, err) - process.exit(0) + process.exit(-1) } } @@ -92,7 +92,7 @@ async function executeMigration (actualVersion: number, entity: { version: strin const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - await sequelizeTypescript.transaction(async t => { + return sequelizeTypescript.transaction(async t => { const options = { transaction: t, queryInterface: sequelizeTypescript.getQueryInterface(), diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts index 94ed1edaa..0779d1e91 100644 --- a/server/lib/activitypub/index.ts +++ b/server/lib/activitypub/index.ts @@ -5,3 +5,8 @@ export * from './fetch' export * from './share' export * from './videos' export * from './url' +export { videoCommentActivityObjectToDBAttributes } from './video-comments' +export { addVideoComments } from './video-comments' +export { addVideoComment } from './video-comments' +export { sendVideoRateChangeToFollowers } from './video-rates' +export { sendVideoRateChangeToOrigin } from './video-rates' diff --git a/server/lib/activitypub/process/misc.ts b/server/lib/activitypub/process/misc.ts deleted file mode 100644 index 461619ea7..000000000 --- a/server/lib/activitypub/process/misc.ts +++ /dev/null @@ -1,194 +0,0 @@ -import * as magnetUtil from 'magnet-uri' -import { VideoTorrentObject } from '../../../../shared' -import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { isVideoFileInfoHashValid } from '../../../helpers/custom-validators/videos' -import { logger } from '../../../helpers/logger' -import { doRequest } from '../../../helpers/requests' -import { ACTIVITY_PUB, VIDEO_MIMETYPE_EXT } from '../../../initializers' -import { ActorModel } from '../../../models/activitypub/actor' -import { VideoModel } from '../../../models/video/video' -import { VideoChannelModel } from '../../../models/video/video-channel' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoShareModel } from '../../../models/video/video-share' -import { getOrCreateActorAndServerAndModel } from '../actor' - -async function videoActivityObjectToDBAttributes ( - videoChannel: VideoChannelModel, - videoObject: VideoTorrentObject, - to: string[] = [], - cc: string[] = [] -) { - let privacy = VideoPrivacy.PRIVATE - if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC - else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED - - const duration = videoObject.duration.replace(/[^\d]+/, '') - let language = null - if (videoObject.language) { - language = parseInt(videoObject.language.identifier, 10) - } - - let category = null - if (videoObject.category) { - category = parseInt(videoObject.category.identifier, 10) - } - - let licence = null - if (videoObject.licence) { - licence = parseInt(videoObject.licence.identifier, 10) - } - - let description = null - if (videoObject.content) { - description = videoObject.content - } - - return { - name: videoObject.name, - uuid: videoObject.uuid, - url: videoObject.id, - category, - licence, - language, - description, - nsfw: videoObject.nsfw, - commentsEnabled: videoObject.commentsEnabled, - channelId: videoChannel.id, - duration: parseInt(duration, 10), - createdAt: new Date(videoObject.published), - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: new Date(videoObject.updated), - views: videoObject.views, - likes: 0, - dislikes: 0, - remote: true, - privacy - } -} - -function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { - const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) - const fileUrls = videoObject.url.filter(u => { - return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/') - }) - - if (fileUrls.length === 0) { - throw new Error('Cannot find video files for ' + videoCreated.url) - } - - const attributes = [] - for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = videoObject.url.find(u => { - return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width - }) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url) - - const parsed = magnetUtil.decode(magnet.url) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url) - - const attribute = { - extname: VIDEO_MIMETYPE_EXT[fileUrl.mimeType], - infoHash: parsed.infoHash, - resolution: fileUrl.width, - size: fileUrl.size, - videoId: videoCreated.id - } - attributes.push(attribute) - } - - return attributes -} - -async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { - let originCommentId: number = null - let inReplyToCommentId: number = null - - // If this is not a reply to the video (thread), create or get the parent comment - if (video.url !== comment.inReplyTo) { - const [ parent ] = await addVideoComment(video, comment.inReplyTo) - if (!parent) { - logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id) - return undefined - } - - originCommentId = parent.originCommentId || parent.id - inReplyToCommentId = parent.id - } - - return { - url: comment.url, - text: comment.content, - videoId: video.id, - accountId: actor.Account.id, - inReplyToCommentId, - originCommentId, - createdAt: new Date(comment.published), - updatedAt: new Date(comment.updated) - } -} - -async function addVideoShares (instance: VideoModel, shareUrls: string[]) { - for (const shareUrl of shareUrls) { - // Fetch url - const { body } = await doRequest({ - uri: shareUrl, - json: true, - activityPub: true - }) - const actorUrl = body.actor - if (!actorUrl) continue - - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - - const entry = { - actorId: actor.id, - videoId: instance.id - } - - await VideoShareModel.findOrCreate({ - where: entry, - defaults: entry - }) - } -} - -async function addVideoComments (instance: VideoModel, commentUrls: string[]) { - for (const commentUrl of commentUrls) { - await addVideoComment(instance, commentUrl) - } -} - -async function addVideoComment (instance: VideoModel, commentUrl: string) { - // Fetch url - const { body } = await doRequest({ - uri: commentUrl, - json: true, - activityPub: true - }) - - const actorUrl = body.attributedTo - if (!actorUrl) return [] - - const actor = await getOrCreateActorAndServerAndModel(actorUrl) - const entry = await videoCommentActivityObjectToDBAttributes(instance, actor, body) - if (!entry) return [] - - return VideoCommentModel.findOrCreate({ - where: { - url: body.id - }, - defaults: entry - }) -} - -// --------------------------------------------------------------------------- - -export { - videoFileActivityUrlToDBAttributes, - videoActivityObjectToDBAttributes, - addVideoShares, - addVideoComments -} diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 9adb40e01..bf7d7879d 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -7,6 +7,7 @@ import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity } from '../send/misc' +import { getOrCreateAccountAndVideoAndChannel } from '../videos' import { processCreateActivity } from './process-create' async function processAnnounceActivity (activity: ActivityAnnounce) { @@ -44,19 +45,19 @@ function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnoun return retryTransactionWrapper(shareVideo, options) } -function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { +async function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { const announced = activity.object + let video: VideoModel + + if (typeof announced === 'string') { + const res = await getOrCreateAccountAndVideoAndChannel(announced) + video = res.video + } else { + video = await processCreateActivity(announced) + } return sequelizeTypescript.transaction(async t => { // Add share entry - let video: VideoModel - - if (typeof announced === 'string') { - video = await VideoModel.loadByUrlAndPopulateAccount(announced) - if (!video) throw new Error('Unknown video to share ' + announced) - } else { - video = await processCreateActivity(announced) - } const share = { actorId: actorAnnouncer.id, diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index e65b257c0..08d61996a 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -8,15 +8,13 @@ import { logger } from '../../../helpers/logger' import { sequelizeTypescript } from '../../../initializers' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { TagModel } from '../../../models/video/tag' import { VideoModel } from '../../../models/video/video' import { VideoAbuseModel } from '../../../models/video/video-abuse' import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoFileModel } from '../../../models/video/video-file' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity, getActorsInvolvedInVideo } from '../send/misc' -import { generateThumbnailFromUrl } from '../videos' -import { addVideoComments, addVideoShares, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import { addVideoComments, resolveThread } from '../video-comments' +import { addVideoShares, getOrCreateAccountAndVideoAndChannel } from '../videos' async function processCreateActivity (activity: ActivityCreate) { const activityObject = activity.object @@ -53,17 +51,7 @@ async function processCreateVideo ( ) { const videoToCreateData = activity.object as VideoTorrentObject - const channel = videoToCreateData.attributedTo.find(a => a.type === 'Group') - if (!channel) throw new Error('Cannot find associated video channel to video ' + videoToCreateData.url) - - const channelActor = await getOrCreateActorAndServerAndModel(channel.id) - - const options = { - arguments: [ actor, activity, videoToCreateData, channelActor ], - errorMessage: 'Cannot insert the remote video with many retries.' - } - - const video = await retryTransactionWrapper(createRemoteVideo, options) + const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData, actor) // Process outside the transaction because we could fetch remote data if (videoToCreateData.likes && Array.isArray(videoToCreateData.likes.orderedItems)) { @@ -89,48 +77,6 @@ async function processCreateVideo ( return video } -function createRemoteVideo ( - account: ActorModel, - activity: ActivityCreate, - videoToCreateData: VideoTorrentObject, - channelActor: ActorModel -) { - logger.debug('Adding remote video %s.', videoToCreateData.id) - - return sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - const videoFromDatabase = await VideoModel.loadByUUIDOrURL(videoToCreateData.uuid, videoToCreateData.id, t) - if (videoFromDatabase) return videoFromDatabase - - const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoToCreateData, activity.to, activity.cc) - const video = VideoModel.build(videoData) - - // Don't block on request - generateThumbnailFromUrl(video, videoToCreateData.icon) - .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoToCreateData.id, err)) - - const videoCreated = await video.save(sequelizeOptions) - - const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData) - if (videoFileAttributes.length === 0) { - throw new Error('Cannot find valid files for video %s ' + videoToCreateData.url) - } - - const tasks: Bluebird[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) - await Promise.all(tasks) - - const tags = videoToCreateData.tag.map(t => t.name) - const tagInstances = await TagModel.findOrCreateTags(tags, t) - await videoCreated.$set('Tags', tagInstances, sequelizeOptions) - - logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid) - - return videoCreated - }) -} - async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { let rateCounts = 0 const tasks: Bluebird[] = [] @@ -167,16 +113,15 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea return retryTransactionWrapper(createVideoDislike, options) } -function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { +async function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { const dislike = activity.object as DislikeObject const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t) - if (!video) throw new Error('Unknown video ' + dislike.object) + const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + return sequelizeTypescript.transaction(async t => { const rate = { type: 'dislike' as 'dislike', videoId: video.id, @@ -200,9 +145,7 @@ function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) { async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { const view = activity.object as ViewObject - const video = await VideoModel.loadByUrlAndPopulateAccount(view.object) - - if (!video) throw new Error('Unknown video ' + view.object) + const { video } = await getOrCreateAccountAndVideoAndChannel(view.object) const account = await ActorModel.loadByUrl(view.actor) if (!account) throw new Error('Unknown account ' + view.actor) @@ -225,19 +168,15 @@ function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: Vid return retryTransactionWrapper(addRemoteVideoAbuse, options) } -function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { +async function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) const account = actor.Account if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(videoAbuseToCreateData.object, t) - if (!video) { - logger.warn('Unknown video %s for remote video abuse.', videoAbuseToCreateData.object) - return undefined - } + const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object) + return sequelizeTypescript.transaction(async t => { const videoAbuseData = { reporterAccountId: account.id, reason: videoAbuseToCreateData.content, @@ -259,41 +198,33 @@ function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreat return retryTransactionWrapper(createVideoComment, options) } -function createVideoComment (byActor: ActorModel, activity: ActivityCreate) { +async function createVideoComment (byActor: ActorModel, activity: ActivityCreate) { const comment = activity.object as VideoCommentObject const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) + const { video, parents } = await resolveThread(comment.inReplyTo) + return sequelizeTypescript.transaction(async t => { - let video = await VideoModel.loadByUrlAndPopulateAccount(comment.inReplyTo, t) - let objectToCreate + let originCommentId = null + let inReplyToCommentId = null + + if (parents.length !== 0) { + const parent = parents[0] + + originCommentId = parent.getThreadId() + inReplyToCommentId = parent.id + } // This is a new thread - if (video) { - objectToCreate = { - url: comment.id, - text: comment.content, - originCommentId: null, - inReplyToComment: null, - videoId: video.id, - accountId: byAccount.id - } - } else { - const inReplyToComment = await VideoCommentModel.loadByUrl(comment.inReplyTo, t) - if (!inReplyToComment) throw new Error('Unknown replied comment ' + comment.inReplyTo) - - video = await VideoModel.loadAndPopulateAccount(inReplyToComment.videoId) - - const originCommentId = inReplyToComment.originCommentId || inReplyToComment.id - objectToCreate = { - url: comment.id, - text: comment.content, - originCommentId, - inReplyToCommentId: inReplyToComment.id, - videoId: video.id, - accountId: byAccount.id - } + const objectToCreate = { + url: comment.id, + text: comment.content, + originCommentId, + inReplyToCommentId, + videoId: video.id, + accountId: byAccount.id } const options = { diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 77fadabe1..0d161b126 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts @@ -3,9 +3,9 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils' import { sequelizeTypescript } from '../../../initializers' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' -import { VideoModel } from '../../../models/video/video' import { getOrCreateActorAndServerAndModel } from '../actor' import { forwardActivity } from '../send/misc' +import { getOrCreateAccountAndVideoAndChannel } from '../videos' async function processLikeActivity (activity: ActivityLike) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -30,17 +30,15 @@ async function processLikeVideo (actor: ActorModel, activity: ActivityLike) { return retryTransactionWrapper(createVideoLike, options) } -function createVideoLike (byActor: ActorModel, activity: ActivityLike) { +async function createVideoLike (byActor: ActorModel, activity: ActivityLike) { const videoUrl = activity.object const byAccount = byActor.Account if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) - - if (!video) throw new Error('Unknown video ' + videoUrl) + const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl) + return sequelizeTypescript.transaction(async t => { const rate = { type: 'like' as 'like', videoId: video.id, diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 9cad59233..5a770bb97 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -7,8 +7,8 @@ import { AccountModel } from '../../../models/account/account' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' -import { VideoModel } from '../../../models/video/video' import { forwardActivity } from '../send/misc' +import { getOrCreateAccountAndVideoAndChannel } from '../videos' async function processUndoActivity (activity: ActivityUndo) { const activityToUndo = activity.object @@ -43,16 +43,15 @@ function processUndoLike (actorUrl: string, activity: ActivityUndo) { return retryTransactionWrapper(undoLike, options) } -function undoLike (actorUrl: string, activity: ActivityUndo) { +async function undoLike (actorUrl: string, activity: ActivityUndo) { const likeActivity = activity.object as ActivityLike + const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object) + return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) if (!byAccount) throw new Error('Unknown account ' + actorUrl) - const video = await VideoModel.loadByUrlAndPopulateAccount(likeActivity.object, t) - if (!video) throw new Error('Unknown video ' + likeActivity.actor) - const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) @@ -76,16 +75,15 @@ function processUndoDislike (actorUrl: string, activity: ActivityUndo) { return retryTransactionWrapper(undoDislike, options) } -function undoDislike (actorUrl: string, activity: ActivityUndo) { +async function undoDislike (actorUrl: string, activity: ActivityUndo) { const dislike = activity.object.object as DislikeObject + const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) + return sequelizeTypescript.transaction(async t => { const byAccount = await AccountModel.loadByUrl(actorUrl, t) if (!byAccount) throw new Error('Unknown account ' + actorUrl) - const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t) - if (!video) throw new Error('Unknown video ' + dislike.actor) - const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t) if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 2c094f7ca..a5431c76b 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -9,10 +9,9 @@ import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' import { TagModel } from '../../../models/video/tag' -import { VideoModel } from '../../../models/video/video' import { VideoFileModel } from '../../../models/video/video-file' import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' -import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' +import { getOrCreateAccountAndVideoAndChannel, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from '../videos' async function processUpdateActivity (activity: ActivityUpdate) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -46,8 +45,10 @@ function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) { async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) { const videoAttributesToUpdate = activity.object as VideoTorrentObject + const res = await getOrCreateAccountAndVideoAndChannel(videoAttributesToUpdate.id) + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid) - let videoInstance: VideoModel + let videoInstance = res.video let videoFieldsSave: any try { @@ -56,9 +57,6 @@ async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) { transaction: t } - const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(videoAttributesToUpdate.id, t) - if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.') - videoFieldsSave = videoInstance.toJSON() const videoChannel = videoInstance.VideoChannel diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts new file mode 100644 index 000000000..17c86a381 --- /dev/null +++ b/server/lib/activitypub/video-comments.ts @@ -0,0 +1,156 @@ +import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object' +import { isVideoCommentObjectValid } from '../../helpers/custom-validators/activitypub/video-comments' +import { logger } from '../../helpers/logger' +import { doRequest } from '../../helpers/requests' +import { ACTIVITY_PUB } from '../../initializers' +import { ActorModel } from '../../models/activitypub/actor' +import { VideoModel } from '../../models/video/video' +import { VideoCommentModel } from '../../models/video/video-comment' +import { getOrCreateActorAndServerAndModel } from './actor' +import { getOrCreateAccountAndVideoAndChannel } from './videos' + +async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { + let originCommentId: number = null + let inReplyToCommentId: number = null + + // If this is not a reply to the video (thread), create or get the parent comment + if (video.url !== comment.inReplyTo) { + const [ parent ] = await addVideoComment(video, comment.inReplyTo) + if (!parent) { + logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id) + return undefined + } + + originCommentId = parent.originCommentId || parent.id + inReplyToCommentId = parent.id + } + + return { + url: comment.url, + text: comment.content, + videoId: video.id, + accountId: actor.Account.id, + inReplyToCommentId, + originCommentId, + createdAt: new Date(comment.published), + updatedAt: new Date(comment.updated) + } +} + +async function addVideoComments (instance: VideoModel, commentUrls: string[]) { + for (const commentUrl of commentUrls) { + await addVideoComment(instance, commentUrl) + } +} + +async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { + logger.info('Fetching remote video comment %s.', commentUrl) + + const { body } = await doRequest({ + uri: commentUrl, + json: true, + activityPub: true + }) + + if (isVideoCommentObjectValid(body) === false) { + logger.debug('Remote video comment JSON is not valid.', { body }) + return undefined + } + + const actorUrl = body.attributedTo + if (!actorUrl) return [] + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) + if (!entry) return [] + + return VideoCommentModel.findOrCreate({ + where: { + url: body.id + }, + defaults: entry + }) +} + +async function resolveThread (url: string, comments: VideoCommentModel[] = []) { + // Already have this comment? + const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideo(url) + if (commentFromDatabase) { + let parentComments = comments.concat([ commentFromDatabase ]) + + // Speed up things and resolve directly the thread + if (commentFromDatabase.InReplyToVideoComment) { + const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') + console.log(data) + + parentComments = parentComments.concat(data) + } + + return resolveThread(commentFromDatabase.Video.url, parentComments) + } + + try { + // Maybe it's a reply to a video? + const { video } = await getOrCreateAccountAndVideoAndChannel(url) + + if (comments.length !== 0) { + const firstReply = comments[ comments.length - 1 ] + firstReply.inReplyToCommentId = null + firstReply.originCommentId = null + firstReply.videoId = video.id + comments[comments.length - 1] = await firstReply.save() + + for (let i = comments.length - 2; i >= 0; i--) { + const comment = comments[ i ] + comment.originCommentId = firstReply.id + comment.inReplyToCommentId = comments[ i + 1 ].id + comment.videoId = video.id + + comments[i] = await comment.save() + } + } + + return { video, parents: comments } + } catch (err) { + logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, err) + + if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { + throw new Error('Recursion limit reached when resolving a thread') + } + + const { body } = await doRequest({ + uri: url, + json: true, + activityPub: true + }) + + if (isVideoCommentObjectValid(body) === false) { + throw new Error('Remote video comment JSON is not valid :' + JSON.stringify(body)) + } + + const actorUrl = body.attributedTo + if (!actorUrl) throw new Error('Miss attributed to in comment') + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + const comment = new VideoCommentModel({ + url: body.url, + text: body.content, + videoId: null, + accountId: actor.Account.id, + inReplyToCommentId: null, + originCommentId: null, + createdAt: new Date(body.published), + updatedAt: new Date(body.updated) + }) + + return resolveThread(body.inReplyTo, comments.concat([ comment ])) + } + +} + +export { + videoCommentActivityObjectToDBAttributes, + addVideoComments, + addVideoComment, + resolveThread +} diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts new file mode 100644 index 000000000..1b2958cca --- /dev/null +++ b/server/lib/activitypub/video-rates.ts @@ -0,0 +1,52 @@ +import { Transaction } from 'sequelize' +import { AccountModel } from '../../models/account/account' +import { VideoModel } from '../../models/video/video' +import { + sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin, + sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers +} from './send' + +async function sendVideoRateChangeToFollowers (account: AccountModel, + video: VideoModel, + likes: number, + dislikes: number, + t: Transaction) { + const actor = account.Actor + + // Keep the order: first we undo and then we create + + // Undo Like + if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t) + // Undo Dislike + if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t) + + // Like + if (likes > 0) await sendLikeToVideoFollowers(actor, video, t) + // Dislike + if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t) +} + +async function sendVideoRateChangeToOrigin (account: AccountModel, + video: VideoModel, + likes: number, + dislikes: number, + t: Transaction) { + const actor = account.Actor + + // Keep the order: first we undo and then we create + + // Undo Like + if (likes < 0) await sendUndoLikeToOrigin(actor, video, t) + // Undo Dislike + if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t) + + // Like + if (likes > 0) await sendLikeToOrigin(actor, video, t) + // Dislike + if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t) +} + +export { + sendVideoRateChangeToFollowers, + sendVideoRateChangeToOrigin +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 8bc928b93..708f4a897 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -1,15 +1,23 @@ +import * as Bluebird from 'bluebird' +import * as magnetUtil from 'magnet-uri' import { join } from 'path' import * as request from 'request' -import { Transaction } from 'sequelize' import { ActivityIconObject } from '../../../shared/index' +import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' +import { VideoPrivacy } from '../../../shared/models/videos' +import { isVideoTorrentObjectValid } from '../../helpers/custom-validators/activitypub/videos' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' +import { retryTransactionWrapper } from '../../helpers/database-utils' +import { logger } from '../../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { CONFIG, REMOTE_SCHEME, STATIC_PATHS } from '../../initializers' -import { AccountModel } from '../../models/account/account' +import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, STATIC_PATHS, VIDEO_MIMETYPE_EXT } from '../../initializers' +import { ActorModel } from '../../models/activitypub/actor' +import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' -import { - sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin, - sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers -} from './send' +import { VideoChannelModel } from '../../models/video/video-channel' +import { VideoFileModel } from '../../models/video/video-file' +import { VideoShareModel } from '../../models/video/video-share' +import { getOrCreateActorAndServerAndModel } from './actor' function fetchRemoteVideoPreview (video: VideoModel, reject: Function) { // FIXME: use url @@ -45,54 +53,221 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) return doRequestAndSaveToFile(options, thumbnailPath) } -async function sendVideoRateChangeToFollowers ( - account: AccountModel, - video: VideoModel, - likes: number, - dislikes: number, - t: Transaction -) { - const actor = account.Actor - - // Keep the order: first we undo and then we create - - // Undo Like - if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t) - // Undo Dislike - if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t) - - // Like - if (likes > 0) await sendLikeToVideoFollowers(actor, video, t) - // Dislike - if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t) +async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelModel, + videoObject: VideoTorrentObject, + to: string[] = [], + cc: string[] = []) { + let privacy = VideoPrivacy.PRIVATE + if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC + else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED + + const duration = videoObject.duration.replace(/[^\d]+/, '') + let language = null + if (videoObject.language) { + language = parseInt(videoObject.language.identifier, 10) + } + + let category = null + if (videoObject.category) { + category = parseInt(videoObject.category.identifier, 10) + } + + let licence = null + if (videoObject.licence) { + licence = parseInt(videoObject.licence.identifier, 10) + } + + let description = null + if (videoObject.content) { + description = videoObject.content + } + + return { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category, + licence, + language, + description, + nsfw: videoObject.nsfw, + commentsEnabled: videoObject.commentsEnabled, + channelId: videoChannel.id, + duration: parseInt(duration, 10), + createdAt: new Date(videoObject.published), + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: new Date(videoObject.updated), + views: videoObject.views, + likes: 0, + dislikes: 0, + remote: true, + privacy + } +} + +function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { + const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) + const fileUrls = videoObject.url.filter(u => { + return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/') + }) + + if (fileUrls.length === 0) { + throw new Error('Cannot find video files for ' + videoCreated.url) + } + + const attributes = [] + for (const fileUrl of fileUrls) { + // Fetch associated magnet uri + const magnet = videoObject.url.find(u => { + return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width + }) + + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url) + + const parsed = magnetUtil.decode(magnet.url) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url) + + const attribute = { + extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], + infoHash: parsed.infoHash, + resolution: fileUrl.width, + size: fileUrl.size, + videoId: videoCreated.id + } + attributes.push(attribute) + } + + return attributes +} + +async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) { + logger.debug('Adding remote video %s.', videoObject.id) + + return sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t) + if (videoFromDatabase) return videoFromDatabase + + const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to, videoObject.cc) + const video = VideoModel.build(videoData) + + // Don't block on request + generateThumbnailFromUrl(video, videoObject.icon) + .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, err)) + + const videoCreated = await video.save(sequelizeOptions) + + const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) + if (videoFileAttributes.length === 0) { + throw new Error('Cannot find valid files for video %s ' + videoObject.url) + } + + const tasks: Bluebird[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) + await Promise.all(tasks) + + const tags = videoObject.tag.map(t => t.name) + const tagInstances = await TagModel.findOrCreateTags(tags, t) + await videoCreated.$set('Tags', tagInstances, sequelizeOptions) + + logger.info('Remote video with uuid %s inserted.', videoObject.uuid) + + videoCreated.VideoChannel = channelActor.VideoChannel + return videoCreated + }) } -async function sendVideoRateChangeToOrigin ( - account: AccountModel, - video: VideoModel, - likes: number, - dislikes: number, - t: Transaction -) { - const actor = account.Actor - - // Keep the order: first we undo and then we create - - // Undo Like - if (likes < 0) await sendUndoLikeToOrigin(actor, video, t) - // Undo Dislike - if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t) - - // Like - if (likes > 0) await sendLikeToOrigin(actor, video, t) - // Dislike - if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t) +async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) { + if (typeof videoObject === 'string') { + const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoObject) + if (videoFromDatabase) { + return { + video: videoFromDatabase, + actor: videoFromDatabase.VideoChannel.Account.Actor, + channelActor: videoFromDatabase.VideoChannel.Actor + } + } + + videoObject = await fetchRemoteVideo(videoObject) + if (!videoObject) throw new Error('Cannot fetch remote video') + } + + if (!actor) { + const actorObj = videoObject.attributedTo.find(a => a.type === 'Person') + if (!actorObj) throw new Error('Cannot find associated actor to video ' + videoObject.url) + + actor = await getOrCreateActorAndServerAndModel(actorObj.id) + } + + const channel = videoObject.attributedTo.find(a => a.type === 'Group') + if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) + + const channelActor = await getOrCreateActorAndServerAndModel(channel.id) + + const options = { + arguments: [ videoObject, channelActor ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + const video = await retryTransactionWrapper(getOrCreateVideo, options) + + return { actor, channelActor, video } +} + +async function addVideoShares (instance: VideoModel, shareUrls: string[]) { + for (const shareUrl of shareUrls) { + // Fetch url + const { body } = await doRequest({ + uri: shareUrl, + json: true, + activityPub: true + }) + const actorUrl = body.actor + if (!actorUrl) continue + + const actor = await getOrCreateActorAndServerAndModel(actorUrl) + + const entry = { + actorId: actor.id, + videoId: instance.id + } + + await VideoShareModel.findOrCreate({ + where: entry, + defaults: entry + }) + } } export { + getOrCreateAccountAndVideoAndChannel, fetchRemoteVideoPreview, fetchRemoteVideoDescription, generateThumbnailFromUrl, - sendVideoRateChangeToFollowers, - sendVideoRateChangeToOrigin + videoActivityObjectToDBAttributes, + videoFileActivityUrlToDBAttributes, + getOrCreateVideo, + addVideoShares} + +// --------------------------------------------------------------------------- + +async function fetchRemoteVideo (videoUrl: string): Promise { + const options = { + uri: videoUrl, + method: 'GET', + json: true, + activityPub: true + } + + logger.info('Fetching remote video %s.', videoUrl) + + const { body } = await doRequest(options) + + if (isVideoTorrentObjectValid(body) === false) { + logger.debug('Remote video JSON is not valid.', { body }) + return undefined + } + + return body } diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 707f140af..b88e06b41 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -62,6 +62,9 @@ enum ScopeNames { @Table({ tableName: 'actor', indexes: [ + { + fields: [ 'url' ] + }, { fields: [ 'preferredUsername', 'serverId' ], unique: true diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index dbb2fe429..fffa4bb57 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -208,7 +208,7 @@ export class VideoCommentModel extends Model { .findOne(query) } - static loadByUrl (url: string, t?: Sequelize.Transaction) { + static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { const query: IFindOptions = { where: { url @@ -217,10 +217,10 @@ export class VideoCommentModel extends Model { if (t !== undefined) query.transaction = t - return VideoCommentModel.findOne(query) + return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query) } - static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) { + static loadByUrlAndPopulateReplyAndVideo (url: string, t?: Sequelize.Transaction) { const query: IFindOptions = { where: { url @@ -229,7 +229,7 @@ export class VideoCommentModel extends Model { if (t !== undefined) query.transaction = t - return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query) + return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_VIDEO ]).findOne(query) } static listThreadsForApi (videoId: number, start: number, count: number, sort: string) { @@ -271,9 +271,9 @@ export class VideoCommentModel extends Model { }) } - static listThreadParentComments (comment: VideoCommentModel, t: Sequelize.Transaction) { + static listThreadParentComments (comment: VideoCommentModel, t: Sequelize.Transaction, order: 'ASC' | 'DESC' = 'ASC') { const query = { - order: [ [ 'createdAt', 'ASC' ] ], + order: [ [ 'createdAt', order ] ], where: { [ Sequelize.Op.or ]: [ { id: comment.getThreadId() }, @@ -281,6 +281,9 @@ export class VideoCommentModel extends Model { ], id: { [ Sequelize.Op.ne ]: comment.id + }, + createdAt: { + [ Sequelize.Op.lt ]: comment.createdAt } }, transaction: t diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 918892938..6b825bf93 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -178,6 +178,10 @@ enum ScopeNames { }, { fields: [ 'id', 'privacy' ] + }, + { + fields: [ 'url'], + unique: true } ] }) @@ -535,7 +539,7 @@ export class VideoModel extends Model { return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) } - static loadByUUIDOrURL (uuid: string, url: string, t?: Sequelize.Transaction) { + static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) { const query: IFindOptions = { where: { [Sequelize.Op.or]: [ @@ -547,7 +551,7 @@ export class VideoModel extends Model { if (t !== undefined) query.transaction = t - return VideoModel.scope(ScopeNames.WITH_FILES).findOne(query) + return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) } static loadAndPopulateAccountAndServerAndTags (id: number) { @@ -983,6 +987,10 @@ export class VideoModel extends Model { { type: 'Group', id: this.VideoChannel.Actor.url + }, + { + type: 'Person', + id: this.VideoChannel.Account.Actor.url } ] } diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts new file mode 100644 index 000000000..da134c09f --- /dev/null +++ b/server/tests/api/server/handle-down.ts @@ -0,0 +1,147 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { VideoPrivacy } from '../../../../shared/models/videos' +import { completeVideoCheck, runServer, viewVideo } from '../../utils' + +import { + flushAndRunMultipleServers, flushTests, getVideosList, killallServers, ServerInfo, setAccessTokensToServers, uploadVideo, + wait +} from '../../utils/index' +import { follow, getFollowersListPaginationAndSort } from '../../utils/server/follows' +import { getJobsListPaginationAndSort } from '../../utils/server/jobs' + +const expect = chai.expect + +describe('Test handle downs', function () { + let servers: ServerInfo[] = [] + + const videoAttributes = { + name: 'my super name for server 1', + category: 5, + licence: 4, + language: 9, + nsfw: true, + description: 'my super description for server 1', + tags: [ 'tag1p1', 'tag2p1' ], + fixture: 'video_short1.webm' + } + + const checkAttributes = { + name: 'my super name for server 1', + category: 5, + licence: 4, + language: 9, + nsfw: true, + description: 'my super description for server 1', + host: 'localhost:9001', + account: 'root', + isLocal: false, + duration: 10, + tags: [ 'tag1p1', 'tag2p1' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + channel: { + name: 'Default root channel', + description: '', + isLocal: false + }, + fixture: 'video_short1.webm', + files: [ + { + resolution: 720, + size: 572456 + } + ] + } + + before(async function () { + this.timeout(20000) + + servers = await flushAndRunMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + }) + + it('Should remove followers that are often down', async function () { + this.timeout(60000) + + await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken) + + await wait(5000) + + await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes) + + await wait(5000) + + for (const server of servers) { + const res = await getVideosList(server.url) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(1) + } + + // Kill server 1 + killallServers([ servers[1] ]) + + // Remove server 2 follower + for (let i = 0; i < 10; i++) { + await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoAttributes) + } + + await wait(10000) + + const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 1, 'createdAt') + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should not have pending/processing jobs anymore', async function () { + const res = await getJobsListPaginationAndSort(servers[0].url, servers[0].accessToken, 0, 50, '-createdAt') + const jobs = res.body.data + + for (const job of jobs) { + expect(job.state).not.to.equal('pending') + expect(job.state).not.to.equal('processing') + } + }) + + it('Should follow server 1', async function () { + servers[1] = await runServer(2) + + await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken) + + await wait(5000) + + const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 1, 'createdAt') + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(1) + }) + + it('Should send a view to server 2, and automatically fetch the video', async function () { + const resVideo = await getVideosList(servers[0].url) + const videoServer1 = resVideo.body.data[0] + + await viewVideo(servers[0].url, videoServer1.uuid) + + await wait(5000) + + const res = await getVideosList(servers[1].url) + const videoServer2 = res.body.data.find(v => v.url === videoServer1.url) + + expect(videoServer2).not.to.be.undefined + + await completeVideoCheck(servers[1].url, videoServer2, checkAttributes) + + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-torrent-object.ts index cf0e0ba54..d3b5f7c26 100644 --- a/shared/models/activitypub/objects/video-torrent-object.ts +++ b/shared/models/activitypub/objects/video-torrent-object.ts @@ -30,4 +30,6 @@ export interface VideoTorrentObject { shares?: ActivityPubOrderedCollection comments?: ActivityPubOrderedCollection attributedTo: ActivityPubAttributedTo[] + to?: string[] + cc?: string[] } -- 2.41.0