From 892211e8493b1f992fce7616cb1e48b7ff87a1dc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 20 Nov 2017 10:24:29 +0100 Subject: Move activitypub functions from helpers/ to lib/ --- server/lib/activitypub/account.ts | 104 +++++++++++++++++++++ server/lib/activitypub/index.ts | 5 + server/lib/activitypub/process/process-add.ts | 17 ++-- server/lib/activitypub/process/process-announce.ts | 2 +- server/lib/activitypub/process/process-create.ts | 3 +- server/lib/activitypub/process/process-delete.ts | 2 +- server/lib/activitypub/process/process-follow.ts | 3 +- server/lib/activitypub/process/process-update.ts | 2 +- server/lib/activitypub/send/send-accept.ts | 2 +- server/lib/activitypub/send/send-announce.ts | 2 +- server/lib/activitypub/send/send-create.ts | 2 +- server/lib/activitypub/send/send-follow.ts | 2 +- server/lib/activitypub/send/send-undo.ts | 2 +- server/lib/activitypub/send/send-update.ts | 2 +- server/lib/activitypub/share.ts | 33 +++++++ server/lib/activitypub/url.ts | 60 ++++++++++++ server/lib/activitypub/video-channels.ts | 60 ++++++++++++ server/lib/activitypub/videos.ts | 44 +++++++++ 18 files changed, 328 insertions(+), 19 deletions(-) create mode 100644 server/lib/activitypub/account.ts create mode 100644 server/lib/activitypub/share.ts create mode 100644 server/lib/activitypub/url.ts create mode 100644 server/lib/activitypub/video-channels.ts create mode 100644 server/lib/activitypub/videos.ts (limited to 'server/lib/activitypub') diff --git a/server/lib/activitypub/account.ts b/server/lib/activitypub/account.ts new file mode 100644 index 000000000..704a92e13 --- /dev/null +++ b/server/lib/activitypub/account.ts @@ -0,0 +1,104 @@ +import * as url from 'url' +import { ActivityPubActor } from '../../../shared/models/activitypub/activitypub-actor' +import { isRemoteAccountValid } from '../../helpers/custom-validators/activitypub/account' +import { logger } from '../../helpers/logger' +import { doRequest } from '../../helpers/requests' +import { ACTIVITY_PUB } from '../../initializers/constants' +import { database as db } from '../../initializers/database' + +async function getOrCreateAccount (accountUrl: string) { + let account = await db.Account.loadByUrl(accountUrl) + + // We don't have this account in our database, fetch it on remote + if (!account) { + const res = await fetchRemoteAccountAndCreateServer(accountUrl) + if (res === undefined) throw new Error('Cannot fetch remote account.') + + // Save our new account in database + account = await res.account.save() + } + + return account +} + +async function fetchRemoteAccountAndCreateServer (accountUrl: string) { + const options = { + uri: accountUrl, + method: 'GET', + headers: { + 'Accept': ACTIVITY_PUB.ACCEPT_HEADER + } + } + + logger.info('Fetching remote account %s.', accountUrl) + + let requestResult + try { + requestResult = await doRequest(options) + } catch (err) { + logger.warn('Cannot fetch remote account %s.', accountUrl, err) + return undefined + } + + const accountJSON: ActivityPubActor = JSON.parse(requestResult.body) + if (isRemoteAccountValid(accountJSON) === false) { + logger.debug('Remote account JSON is not valid.', { accountJSON }) + return undefined + } + + const followersCount = await fetchAccountCount(accountJSON.followers) + const followingCount = await fetchAccountCount(accountJSON.following) + + const account = db.Account.build({ + uuid: accountJSON.uuid, + name: accountJSON.preferredUsername, + url: accountJSON.url, + publicKey: accountJSON.publicKey.publicKeyPem, + privateKey: null, + followersCount: followersCount, + followingCount: followingCount, + inboxUrl: accountJSON.inbox, + outboxUrl: accountJSON.outbox, + sharedInboxUrl: accountJSON.endpoints.sharedInbox, + followersUrl: accountJSON.followers, + followingUrl: accountJSON.following + }) + + const accountHost = url.parse(account.url).host + const serverOptions = { + where: { + host: accountHost + }, + defaults: { + host: accountHost + } + } + const [ server ] = await db.Server.findOrCreate(serverOptions) + account.set('serverId', server.id) + + return { account, server } +} + +export { + getOrCreateAccount, + fetchRemoteAccountAndCreateServer +} + +// --------------------------------------------------------------------------- + +async function fetchAccountCount (url: string) { + const options = { + uri: url, + method: 'GET' + } + + let requestResult + try { + requestResult = await doRequest(options) + } catch (err) { + logger.warn('Cannot fetch remote account count %s.', url, err) + return undefined + } + + return requestResult.totalItems ? requestResult.totalItems : 0 +} diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts index 1bea0a412..b1daa70bb 100644 --- a/server/lib/activitypub/index.ts +++ b/server/lib/activitypub/index.ts @@ -1,2 +1,7 @@ export * from './process' export * from './send' +export * from './account' +export * from './share' +export * from './video-channels' +export * from './videos' +export * from './url' diff --git a/server/lib/activitypub/process/process-add.ts b/server/lib/activitypub/process/process-add.ts index f064c1ab6..281036228 100644 --- a/server/lib/activitypub/process/process-add.ts +++ b/server/lib/activitypub/process/process-add.ts @@ -1,11 +1,14 @@ import * as Bluebird from 'bluebird' import { VideoTorrentObject } from '../../../../shared' import { ActivityAdd } from '../../../../shared/models/activitypub/activity' -import { generateThumbnailFromUrl, getOrCreateAccount, logger, retryTransactionWrapper } from '../../../helpers' -import { getOrCreateVideoChannel } from '../../../helpers/activitypub' +import { retryTransactionWrapper } from '../../../helpers/database-utils' +import { logger } from '../../../helpers/logger' import { database as db } from '../../../initializers' import { AccountInstance } from '../../../models/account/account-interface' import { VideoChannelInstance } from '../../../models/video/video-channel-interface' +import { getOrCreateAccount } from '../account' +import { getOrCreateVideoChannel } from '../video-channels' +import { generateThumbnailFromUrl } from '../videos' import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' async function processAddActivity (activity: ActivityAdd) { @@ -41,12 +44,10 @@ function processAddVideo (account: AccountInstance, activity: ActivityAdd, video return retryTransactionWrapper(addRemoteVideo, options) } -function addRemoteVideo ( - account: AccountInstance, - activity: ActivityAdd, - videoChannel: VideoChannelInstance, - videoToCreateData: VideoTorrentObject -) { +function addRemoteVideo (account: AccountInstance, + activity: ActivityAdd, + videoChannel: VideoChannelInstance, + videoToCreateData: VideoTorrentObject) { logger.debug('Adding remote video %s.', videoToCreateData.url) return db.sequelize.transaction(async t => { diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 656db08a9..40712ef03 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts @@ -1,11 +1,11 @@ import { ActivityAnnounce } from '../../../../shared/models/activitypub/activity' -import { getOrCreateAccount } from '../../../helpers/activitypub' import { logger } from '../../../helpers/logger' import { database as db } from '../../../initializers/index' import { VideoInstance } from '../../../models/index' import { VideoChannelInstance } from '../../../models/video/video-channel-interface' import { processAddActivity } from './process-add' import { processCreateActivity } from './process-create' +import { getOrCreateAccount } from '../account' async function processAnnounceActivity (activity: ActivityAnnounce) { const announcedActivity = activity.object diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index aac941a6c..fc635eb1f 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -1,9 +1,10 @@ import { ActivityCreate, VideoChannelObject } from '../../../../shared' import { VideoAbuseObject } from '../../../../shared/models/activitypub/objects/video-abuse-object' import { logger, retryTransactionWrapper } from '../../../helpers' -import { getOrCreateAccount, getVideoChannelActivityPubUrl } from '../../../helpers/activitypub' import { database as db } from '../../../initializers' import { AccountInstance } from '../../../models/account/account-interface' +import { getOrCreateAccount } from '../account' +import { getVideoChannelActivityPubUrl } from '../url' import { videoChannelActivityObjectToDBAttributes } from './misc' async function processCreateActivity (activity: ActivityCreate) { diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts index af5d964d4..0328d1a7d 100644 --- a/server/lib/activitypub/process/process-delete.ts +++ b/server/lib/activitypub/process/process-delete.ts @@ -1,11 +1,11 @@ import { ActivityDelete } from '../../../../shared/models/activitypub/activity' -import { getOrCreateAccount } from '../../../helpers/activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { database as db } from '../../../initializers' import { AccountInstance } from '../../../models/account/account-interface' import { VideoChannelInstance } from '../../../models/video/video-channel-interface' import { VideoInstance } from '../../../models/video/video-interface' +import { getOrCreateAccount } from '../account' async function processDeleteActivity (activity: ActivityDelete) { const account = await getOrCreateAccount(activity.actor) diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts index 553639580..41b38828c 100644 --- a/server/lib/activitypub/process/process-follow.ts +++ b/server/lib/activitypub/process/process-follow.ts @@ -1,9 +1,10 @@ import { ActivityFollow } from '../../../../shared/models/activitypub/activity' -import { getOrCreateAccount, retryTransactionWrapper } from '../../../helpers' +import { retryTransactionWrapper } from '../../../helpers' import { database as db } from '../../../initializers' import { AccountInstance } from '../../../models/account/account-interface' import { logger } from '../../../helpers/logger' import { sendAccept } from '../send/send-accept' +import { getOrCreateAccount } from '../account' async function processFollowActivity (activity: ActivityFollow) { const activityObject = activity.object diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index a3bfb1baf..4876735b8 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -1,6 +1,5 @@ import { VideoChannelObject, VideoTorrentObject } from '../../../../shared' import { ActivityUpdate } from '../../../../shared/models/activitypub/activity' -import { getOrCreateAccount } from '../../../helpers/activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' import { resetSequelizeInstance } from '../../../helpers/utils' @@ -9,6 +8,7 @@ import { AccountInstance } from '../../../models/account/account-interface' import { VideoInstance } from '../../../models/video/video-interface' import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' import Bluebird = require('bluebird') +import { getOrCreateAccount } from '../account' async function processUpdateActivity (activity: ActivityUpdate) { const account = await getOrCreateAccount(activity.actor) diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts index 0324a30fa..eeab19ed0 100644 --- a/server/lib/activitypub/send/send-accept.ts +++ b/server/lib/activitypub/send/send-accept.ts @@ -3,7 +3,7 @@ import { ActivityAccept } from '../../../../shared/models/activitypub/activity' import { AccountInstance } from '../../../models' import { AccountFollowInstance } from '../../../models/account/account-follow-interface' import { unicastTo } from './misc' -import { getAccountFollowAcceptActivityPubUrl } from '../../../helpers/activitypub' +import { getAccountFollowAcceptActivityPubUrl } from '../url' async function sendAccept (accountFollow: AccountFollowInstance, t: Transaction) { const follower = accountFollow.AccountFollower diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index b9217e4f6..4b3a4ef75 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts @@ -4,7 +4,7 @@ import { VideoChannelInstance } from '../../../models/video/video-channel-interf import { broadcastToFollowers } from './misc' import { addActivityData } from './send-add' import { createActivityData } from './send-create' -import { getAnnounceActivityPubUrl } from '../../../helpers/activitypub' +import { getAnnounceActivityPubUrl } from '../url' async function sendVideoAnnounce (byAccount: AccountInstance, video: VideoInstance, t: Transaction) { const url = getAnnounceActivityPubUrl(video.url, byAccount) diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 66bfeee89..14b666fc9 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -3,7 +3,7 @@ import { ActivityCreate } from '../../../../shared/models/activitypub/activity' import { AccountInstance, VideoChannelInstance, VideoInstance } from '../../../models' import { VideoAbuseInstance } from '../../../models/video/video-abuse-interface' import { broadcastToFollowers, getAudience, unicastTo } from './misc' -import { getVideoAbuseActivityPubUrl } from '../../../helpers/activitypub' +import { getVideoAbuseActivityPubUrl } from '../url' async function sendCreateVideoChannel (videoChannel: VideoChannelInstance, t: Transaction) { const byAccount = videoChannel.Account diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts index 48d641c22..3c01fb77c 100644 --- a/server/lib/activitypub/send/send-follow.ts +++ b/server/lib/activitypub/send/send-follow.ts @@ -2,8 +2,8 @@ import { Transaction } from 'sequelize' import { ActivityFollow } from '../../../../shared/models/activitypub/activity' import { AccountInstance } from '../../../models' import { AccountFollowInstance } from '../../../models/account/account-follow-interface' +import { getAccountFollowActivityPubUrl } from '../url' import { unicastTo } from './misc' -import { getAccountFollowActivityPubUrl } from '../../../helpers/activitypub' async function sendFollow (accountFollow: AccountFollowInstance, t: Transaction) { const me = accountFollow.AccountFollower diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 39da824f3..77bee6639 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -3,8 +3,8 @@ import { ActivityFollow, ActivityUndo } from '../../../../shared/models/activity import { AccountInstance } from '../../../models' import { AccountFollowInstance } from '../../../models/account/account-follow-interface' import { unicastTo } from './misc' -import { getAccountFollowActivityPubUrl, getUndoActivityPubUrl } from '../../../helpers/activitypub' import { followActivityData } from './send-follow' +import { getAccountFollowActivityPubUrl, getUndoActivityPubUrl } from '../url' async function sendUndoFollow (accountFollow: AccountFollowInstance, t: Transaction) { const me = accountFollow.AccountFollower diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 42738f973..32cada06e 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts @@ -1,8 +1,8 @@ import { Transaction } from 'sequelize' import { ActivityUpdate } from '../../../../shared/models/activitypub/activity' -import { getUpdateActivityPubUrl } from '../../../helpers/activitypub' import { database as db } from '../../../initializers' import { AccountInstance, VideoChannelInstance, VideoInstance } from '../../../models' +import { getUpdateActivityPubUrl } from '../url' import { broadcastToFollowers, getAudience } from './misc' async function sendUpdateVideoChannel (videoChannel: VideoChannelInstance, t: Transaction) { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts new file mode 100644 index 000000000..689e200a6 --- /dev/null +++ b/server/lib/activitypub/share.ts @@ -0,0 +1,33 @@ +import { Transaction } from 'sequelize' +import { getServerAccount } from '../../helpers/utils' +import { database as db } from '../../initializers' +import { VideoChannelInstance } from '../../models/index' +import { VideoInstance } from '../../models/video/video-interface' +import { sendVideoAnnounce, sendVideoChannelAnnounce } from './send/send-announce' + +async function shareVideoChannelByServer (videoChannel: VideoChannelInstance, t: Transaction) { + const serverAccount = await getServerAccount() + + await db.VideoChannelShare.create({ + accountId: serverAccount.id, + videoChannelId: videoChannel.id + }, { transaction: t }) + + return sendVideoChannelAnnounce(serverAccount, videoChannel, t) +} + +async function shareVideoByServer (video: VideoInstance, t: Transaction) { + const serverAccount = await getServerAccount() + + await db.VideoShare.create({ + accountId: serverAccount.id, + videoId: video.id + }, { transaction: t }) + + return sendVideoAnnounce(serverAccount, video, t) +} + +export { + shareVideoChannelByServer, + shareVideoByServer +} diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts new file mode 100644 index 000000000..41ac0f9a8 --- /dev/null +++ b/server/lib/activitypub/url.ts @@ -0,0 +1,60 @@ +import { CONFIG } from '../../initializers/constants' +import { VideoInstance } from '../../models/video/video-interface' +import { VideoChannelInstance } from '../../models/video/video-channel-interface' +import { VideoAbuseInstance } from '../../models/video/video-abuse-interface' +import { AccountFollowInstance } from '../../models/account/account-follow-interface' +import { AccountInstance } from '../../models/account/account-interface' + +function getVideoActivityPubUrl (video: VideoInstance) { + return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid +} + +function getVideoChannelActivityPubUrl (videoChannel: VideoChannelInstance) { + return CONFIG.WEBSERVER.URL + '/video-channels/' + videoChannel.uuid +} + +function getAccountActivityPubUrl (accountName: string) { + return CONFIG.WEBSERVER.URL + '/account/' + accountName +} + +function getVideoAbuseActivityPubUrl (videoAbuse: VideoAbuseInstance) { + return CONFIG.WEBSERVER.URL + '/admin/video-abuses/' + videoAbuse.id +} + +function getAccountFollowActivityPubUrl (accountFollow: AccountFollowInstance) { + const me = accountFollow.AccountFollower + const following = accountFollow.AccountFollowing + + return me.url + '#follows/' + following.id +} + +function getAccountFollowAcceptActivityPubUrl (accountFollow: AccountFollowInstance) { + const follower = accountFollow.AccountFollower + const me = accountFollow.AccountFollowing + + return follower.url + '#accepts/follows/' + me.id +} + +function getAnnounceActivityPubUrl (originalUrl: string, byAccount: AccountInstance) { + return originalUrl + '#announces/' + byAccount.id +} + +function getUpdateActivityPubUrl (originalUrl: string, updatedAt: string) { + return originalUrl + '#updates/' + updatedAt +} + +function getUndoActivityPubUrl (originalUrl: string) { + return originalUrl + '/undo' +} + +export { + getVideoActivityPubUrl, + getVideoChannelActivityPubUrl, + getAccountActivityPubUrl, + getVideoAbuseActivityPubUrl, + getAccountFollowActivityPubUrl, + getAccountFollowAcceptActivityPubUrl, + getAnnounceActivityPubUrl, + getUpdateActivityPubUrl, + getUndoActivityPubUrl +} diff --git a/server/lib/activitypub/video-channels.ts b/server/lib/activitypub/video-channels.ts new file mode 100644 index 000000000..7339d79f9 --- /dev/null +++ b/server/lib/activitypub/video-channels.ts @@ -0,0 +1,60 @@ +import { VideoChannelObject } from '../../../shared/models/activitypub/objects/video-channel-object' +import { isVideoChannelObjectValid } from '../../helpers/custom-validators/activitypub/video-channels' +import { logger } from '../../helpers/logger' +import { doRequest } from '../../helpers/requests' +import { database as db } from '../../initializers' +import { ACTIVITY_PUB } from '../../initializers/constants' +import { AccountInstance } from '../../models/account/account-interface' +import { videoChannelActivityObjectToDBAttributes } from './process/misc' + +async function getOrCreateVideoChannel (ownerAccount: AccountInstance, videoChannelUrl: string) { + let videoChannel = await db.VideoChannel.loadByUrl(videoChannelUrl) + + // We don't have this account in our database, fetch it on remote + if (!videoChannel) { + videoChannel = await fetchRemoteVideoChannel(ownerAccount, videoChannelUrl) + if (videoChannel === undefined) throw new Error('Cannot fetch remote video channel.') + + // Save our new video channel in database + await videoChannel.save() + } + + return videoChannel +} + +async function fetchRemoteVideoChannel (ownerAccount: AccountInstance, videoChannelUrl: string) { + const options = { + uri: videoChannelUrl, + method: 'GET', + headers: { + 'Accept': ACTIVITY_PUB.ACCEPT_HEADER + } + } + + logger.info('Fetching remote video channel %s.', videoChannelUrl) + + let requestResult + try { + requestResult = await doRequest(options) + } catch (err) { + logger.warn('Cannot fetch remote video channel %s.', videoChannelUrl, err) + return undefined + } + + const videoChannelJSON: VideoChannelObject = JSON.parse(requestResult.body) + if (isVideoChannelObjectValid(videoChannelJSON) === false) { + logger.debug('Remote video channel JSON is not valid.', { videoChannelJSON }) + return undefined + } + + const videoChannelAttributes = videoChannelActivityObjectToDBAttributes(videoChannelJSON, ownerAccount) + const videoChannel = db.VideoChannel.build(videoChannelAttributes) + videoChannel.Account = ownerAccount + + return videoChannel +} + +export { + getOrCreateVideoChannel, + fetchRemoteVideoChannel +} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts new file mode 100644 index 000000000..944244893 --- /dev/null +++ b/server/lib/activitypub/videos.ts @@ -0,0 +1,44 @@ +import { join } from 'path' +import * as request from 'request' +import { ActivityIconObject } from '../../../shared/index' +import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' +import { CONFIG, REMOTE_SCHEME, STATIC_PATHS } from '../../initializers/constants' +import { VideoInstance } from '../../models/video/video-interface' + +function fetchRemoteVideoPreview (video: VideoInstance) { + // FIXME: use url + const host = video.VideoChannel.Account.Server.host + const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) + + return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) +} + +async function fetchRemoteVideoDescription (video: VideoInstance) { + // FIXME: use url + const host = video.VideoChannel.Account.Server.host + const path = video.getDescriptionPath() + const options = { + uri: REMOTE_SCHEME.HTTP + '://' + host + path, + json: true + } + + const { body } = await doRequest(options) + return body.description ? body.description : '' +} + +function generateThumbnailFromUrl (video: VideoInstance, icon: ActivityIconObject) { + const thumbnailName = video.getThumbnailName() + const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) + + const options = { + method: 'GET', + uri: icon.url + } + return doRequestAndSaveToFile(options, thumbnailPath) +} + +export { + fetchRemoteVideoPreview, + fetchRemoteVideoDescription, + generateThumbnailFromUrl +} -- cgit v1.2.3