1 import * as Bluebird from 'bluebird'
2 import { Transaction } from 'sequelize'
3 import * as url from 'url'
4 import * as uuidv4 from 'uuid/v4'
5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11 import { logger } from '../../helpers/logger'
12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13 import { doRequest, downloadImage } from '../../helpers/requests'
14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
16 import { AccountModel } from '../../models/account/account'
17 import { ActorModel } from '../../models/activitypub/actor'
18 import { AvatarModel } from '../../models/avatar/avatar'
19 import { ServerModel } from '../../models/server/server'
20 import { VideoChannelModel } from '../../models/video/video-channel'
21 import { JobQueue } from '../job-queue'
22 import { getServerActor } from '../../helpers/utils'
23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
25 // Set account keys, this could be long so process after the account creation and do not block the client
26 function setAsyncActorKeys (actor: ActorModel) {
27 return createPrivateAndPublicKeys()
28 .then(({ publicKey, privateKey }) => {
29 actor.set('publicKey', publicKey)
30 actor.set('privateKey', privateKey)
34 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
39 async function getOrCreateActorAndServerAndModel (
40 activityActor: string | ActivityPubActor,
41 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
42 recurseIfNeeded = true,
43 updateCollections = false
45 const actorUrl = getAPId(activityActor)
48 let actor = await fetchActorByUrl(actorUrl, fetchType)
49 // Orphan actor (not associated to an account of channel) so recreate it
50 if (actor && (!actor.Account && !actor.VideoChannel)) {
55 // We don't have this actor in our database, fetch it on remote
57 const { result } = await fetchRemoteActor(actorUrl)
58 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
60 // Create the attributed to actor
61 // In PeerTube a video channel is owned by an account
62 let ownerActor: ActorModel = undefined
63 if (recurseIfNeeded === true && result.actor.type === 'Group') {
64 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
65 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
67 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
68 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
72 // Don't recurse another time
73 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
75 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
80 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
84 if (actor.Account) actor.Account.Actor = actor
85 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
87 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
88 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
90 if ((created === true || refreshed === true) && updateCollections === true) {
91 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
92 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
98 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
99 return new ActorModel({
108 inboxUrl: url + '/inbox',
109 outboxUrl: url + '/outbox',
110 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
111 followersUrl: url + '/followers',
112 followingUrl: url + '/following'
116 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
117 const followersCount = await fetchActorTotalItems(attributes.followers)
118 const followingCount = await fetchActorTotalItems(attributes.following)
120 actorInstance.set('type', attributes.type)
121 actorInstance.set('uuid', attributes.uuid)
122 actorInstance.set('preferredUsername', attributes.preferredUsername)
123 actorInstance.set('url', attributes.id)
124 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
125 actorInstance.set('followersCount', followersCount)
126 actorInstance.set('followingCount', followingCount)
127 actorInstance.set('inboxUrl', attributes.inbox)
128 actorInstance.set('outboxUrl', attributes.outbox)
129 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
130 actorInstance.set('followersUrl', attributes.followers)
131 actorInstance.set('followingUrl', attributes.following)
134 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
135 if (avatarName !== undefined) {
136 if (actorInstance.avatarId) {
138 await actorInstance.Avatar.destroy({ transaction: t })
140 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
144 const avatar = await AvatarModel.create({
146 }, { transaction: t })
148 actorInstance.set('avatarId', avatar.id)
149 actorInstance.Avatar = avatar
155 async function fetchActorTotalItems (url: string) {
164 const { body } = await doRequest(options)
165 return body.totalItems ? body.totalItems : 0
167 logger.warn('Cannot fetch remote actor count %s.', url, { err })
172 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
174 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
175 isActivityPubUrlValid(actorJSON.icon.url)
177 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
179 const avatarName = uuidv4() + extension
180 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
188 async function addFetchOutboxJob (actor: ActorModel) {
189 // Don't fetch ourselves
190 const serverActor = await getServerActor()
191 if (serverActor.id === actor.id) {
192 logger.error('Cannot fetch our own outbox!')
197 uri: actor.outboxUrl,
198 type: 'activity' as 'activity'
201 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
204 async function refreshActorIfNeeded (
205 actorArg: ActorModel,
206 fetchedType: ActorFetchByUrlType
207 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
208 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
210 // We need more attributes
211 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
216 actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
218 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
222 const { result, statusCode } = await fetchRemoteActor(actorUrl)
224 if (statusCode === 404) {
225 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
226 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
227 return { actor: undefined, refreshed: false }
230 if (result === undefined) {
231 logger.warn('Cannot fetch remote actor in refresh actor.')
232 return { actor, refreshed: false }
235 return sequelizeTypescript.transaction(async t => {
236 updateInstanceWithAnother(actor, result.actor)
238 if (result.avatarName !== undefined) {
239 await updateActorAvatarInstance(actor, result.avatarName, t)
243 actor.setDataValue('updatedAt', new Date())
244 await actor.save({ transaction: t })
247 actor.Account.set('name', result.name)
248 actor.Account.set('description', result.summary)
250 await actor.Account.save({ transaction: t })
251 } else if (actor.VideoChannel) {
252 actor.VideoChannel.set('name', result.name)
253 actor.VideoChannel.set('description', result.summary)
254 actor.VideoChannel.set('support', result.support)
256 await actor.VideoChannel.save({ transaction: t })
259 return { refreshed: true, actor }
262 logger.warn('Cannot refresh actor.', { err })
263 return { actor, refreshed: false }
268 getOrCreateActorAndServerAndModel,
271 fetchActorTotalItems,
274 refreshActorIfNeeded,
275 updateActorAvatarInstance,
279 // ---------------------------------------------------------------------------
281 function saveActorAndServerAndModelIfNotExist (
282 result: FetchRemoteActorResult,
283 ownerActor?: ActorModel,
285 ): Bluebird<ActorModel> | Promise<ActorModel> {
286 let actor = result.actor
288 if (t !== undefined) return save(t)
290 return sequelizeTypescript.transaction(t => save(t))
292 async function save (t: Transaction) {
293 const actorHost = url.parse(actor.url).host
295 const serverOptions = {
304 const [ server ] = await ServerModel.findOrCreate(serverOptions)
306 // Save our new account in database
307 actor.set('serverId', server.id)
310 if (result.avatarName) {
311 const avatar = await AvatarModel.create({
312 filename: result.avatarName
313 }, { transaction: t })
314 actor.set('avatarId', avatar.id)
317 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
318 // (which could be false in a retried query)
319 const [ actorCreated ] = await ActorModel.findOrCreate({
320 defaults: actor.toJSON(),
327 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
328 actorCreated.Account = await saveAccount(actorCreated, result, t)
329 actorCreated.Account.Actor = actorCreated
330 } else if (actorCreated.type === 'Group') { // Video channel
331 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
332 actorCreated.VideoChannel.Actor = actorCreated
333 actorCreated.VideoChannel.Account = ownerActor.Account
340 type FetchRemoteActorResult = {
346 attributedTo: ActivityPubAttributedTo[]
348 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
356 logger.info('Fetching remote actor %s.', actorUrl)
358 const requestResult = await doRequest<ActivityPubActor>(options)
359 normalizeActor(requestResult.body)
361 const actorJSON = requestResult.body
362 if (isActorObjectValid(actorJSON) === false) {
363 logger.debug('Remote actor JSON is not valid.', { actorJSON })
364 return { result: undefined, statusCode: requestResult.response.statusCode }
367 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
368 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
371 const followersCount = await fetchActorTotalItems(actorJSON.followers)
372 const followingCount = await fetchActorTotalItems(actorJSON.following)
374 const actor = new ActorModel({
375 type: actorJSON.type,
376 uuid: actorJSON.uuid,
377 preferredUsername: actorJSON.preferredUsername,
379 publicKey: actorJSON.publicKey.publicKeyPem,
381 followersCount: followersCount,
382 followingCount: followingCount,
383 inboxUrl: actorJSON.inbox,
384 outboxUrl: actorJSON.outbox,
385 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
386 followersUrl: actorJSON.followers,
387 followingUrl: actorJSON.following
390 const avatarName = await fetchAvatarIfExists(actorJSON)
392 const name = actorJSON.name || actorJSON.preferredUsername
394 statusCode: requestResult.response.statusCode,
399 summary: actorJSON.summary,
400 support: actorJSON.support,
401 attributedTo: actorJSON.attributedTo
406 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
407 const [ accountCreated ] = await AccountModel.findOrCreate({
410 description: result.summary,
419 return accountCreated
422 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
423 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
426 description: result.summary,
427 support: result.support,
429 accountId: ownerActor.Account.id
437 return videoChannelCreated