1 import * as Bluebird from 'bluebird'
2 import { join } from 'path'
3 import { Transaction } from 'sequelize'
4 import * as url from 'url'
5 import * as uuidv4 from 'uuid/v4'
6 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
7 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
8 import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
9 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
10 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
11 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
14 import { doRequest, doRequestAndSaveToFile, downloadImage } from '../../helpers/requests'
15 import { getUrlFromWebfinger } from '../../helpers/webfinger'
16 import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript } from '../../initializers'
17 import { AccountModel } from '../../models/account/account'
18 import { ActorModel } from '../../models/activitypub/actor'
19 import { AvatarModel } from '../../models/avatar/avatar'
20 import { ServerModel } from '../../models/server/server'
21 import { VideoChannelModel } from '../../models/video/video-channel'
22 import { JobQueue } from '../job-queue'
23 import { getServerActor } from '../../helpers/utils'
24 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
26 // Set account keys, this could be long so process after the account creation and do not block the client
27 function setAsyncActorKeys (actor: ActorModel) {
28 return createPrivateAndPublicKeys()
29 .then(({ publicKey, privateKey }) => {
30 actor.set('publicKey', publicKey)
31 actor.set('privateKey', privateKey)
35 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
40 async function getOrCreateActorAndServerAndModel (
41 activityActor: string | ActivityPubActor,
42 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
43 recurseIfNeeded = true,
44 updateCollections = false
46 const actorUrl = getActorUrl(activityActor)
49 let actor = await fetchActorByUrl(actorUrl, fetchType)
50 // Orphan actor (not associated to an account of channel) so recreate it
51 if (actor && (!actor.Account && !actor.VideoChannel)) {
56 // We don't have this actor in our database, fetch it on remote
58 const { result } = await fetchRemoteActor(actorUrl)
59 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
61 // Create the attributed to actor
62 // In PeerTube a video channel is owned by an account
63 let ownerActor: ActorModel = undefined
64 if (recurseIfNeeded === true && result.actor.type === 'Group') {
65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
68 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
69 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
73 // Don't recurse another time
74 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
76 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
81 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
85 if (actor.Account) actor.Account.Actor = actor
86 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
88 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
89 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
91 if ((created === true || refreshed === true) && updateCollections === true) {
92 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
93 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
99 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
100 return new ActorModel({
109 inboxUrl: url + '/inbox',
110 outboxUrl: url + '/outbox',
111 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
112 followersUrl: url + '/followers',
113 followingUrl: url + '/following'
117 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
118 const followersCount = await fetchActorTotalItems(attributes.followers)
119 const followingCount = await fetchActorTotalItems(attributes.following)
121 actorInstance.set('type', attributes.type)
122 actorInstance.set('uuid', attributes.uuid)
123 actorInstance.set('preferredUsername', attributes.preferredUsername)
124 actorInstance.set('url', attributes.id)
125 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
126 actorInstance.set('followersCount', followersCount)
127 actorInstance.set('followingCount', followingCount)
128 actorInstance.set('inboxUrl', attributes.inbox)
129 actorInstance.set('outboxUrl', attributes.outbox)
130 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
131 actorInstance.set('followersUrl', attributes.followers)
132 actorInstance.set('followingUrl', attributes.following)
135 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
136 if (avatarName !== undefined) {
137 if (actorInstance.avatarId) {
139 await actorInstance.Avatar.destroy({ transaction: t })
141 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
145 const avatar = await AvatarModel.create({
147 }, { transaction: t })
149 actorInstance.set('avatarId', avatar.id)
150 actorInstance.Avatar = avatar
156 async function fetchActorTotalItems (url: string) {
165 const { body } = await doRequest(options)
166 return body.totalItems ? body.totalItems : 0
168 logger.warn('Cannot fetch remote actor count %s.', url, { err })
173 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
175 actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
176 isActivityPubUrlValid(actorJSON.icon.url)
178 const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
180 const avatarName = uuidv4() + extension
181 const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
183 await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE)
191 async function addFetchOutboxJob (actor: ActorModel) {
192 // Don't fetch ourselves
193 const serverActor = await getServerActor()
194 if (serverActor.id === actor.id) {
195 logger.error('Cannot fetch our own outbox!')
200 uri: actor.outboxUrl,
201 type: 'activity' as 'activity'
204 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
208 getOrCreateActorAndServerAndModel,
211 fetchActorTotalItems,
214 updateActorAvatarInstance,
218 // ---------------------------------------------------------------------------
220 function saveActorAndServerAndModelIfNotExist (
221 result: FetchRemoteActorResult,
222 ownerActor?: ActorModel,
224 ): Bluebird<ActorModel> | Promise<ActorModel> {
225 let actor = result.actor
227 if (t !== undefined) return save(t)
229 return sequelizeTypescript.transaction(t => save(t))
231 async function save (t: Transaction) {
232 const actorHost = url.parse(actor.url).host
234 const serverOptions = {
243 const [ server ] = await ServerModel.findOrCreate(serverOptions)
245 // Save our new account in database
246 actor.set('serverId', server.id)
249 if (result.avatarName) {
250 const avatar = await AvatarModel.create({
251 filename: result.avatarName
252 }, { transaction: t })
253 actor.set('avatarId', avatar.id)
256 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
257 // (which could be false in a retried query)
258 const [ actorCreated ] = await ActorModel.findOrCreate({
259 defaults: actor.toJSON(),
266 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
267 actorCreated.Account = await saveAccount(actorCreated, result, t)
268 actorCreated.Account.Actor = actorCreated
269 } else if (actorCreated.type === 'Group') { // Video channel
270 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
271 actorCreated.VideoChannel.Actor = actorCreated
272 actorCreated.VideoChannel.Account = ownerActor.Account
279 type FetchRemoteActorResult = {
285 attributedTo: ActivityPubAttributedTo[]
287 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
295 logger.info('Fetching remote actor %s.', actorUrl)
297 const requestResult = await doRequest(options)
298 normalizeActor(requestResult.body)
300 const actorJSON: ActivityPubActor = requestResult.body
301 if (isActorObjectValid(actorJSON) === false) {
302 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
303 return { result: undefined, statusCode: requestResult.response.statusCode }
306 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
307 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
310 const followersCount = await fetchActorTotalItems(actorJSON.followers)
311 const followingCount = await fetchActorTotalItems(actorJSON.following)
313 const actor = new ActorModel({
314 type: actorJSON.type,
315 uuid: actorJSON.uuid,
316 preferredUsername: actorJSON.preferredUsername,
318 publicKey: actorJSON.publicKey.publicKeyPem,
320 followersCount: followersCount,
321 followingCount: followingCount,
322 inboxUrl: actorJSON.inbox,
323 outboxUrl: actorJSON.outbox,
324 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
325 followersUrl: actorJSON.followers,
326 followingUrl: actorJSON.following
329 const avatarName = await fetchAvatarIfExists(actorJSON)
331 const name = actorJSON.name || actorJSON.preferredUsername
333 statusCode: requestResult.response.statusCode,
338 summary: actorJSON.summary,
339 support: actorJSON.support,
340 attributedTo: actorJSON.attributedTo
345 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
346 const [ accountCreated ] = await AccountModel.findOrCreate({
349 description: result.summary,
358 return accountCreated
361 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
362 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
365 description: result.summary,
366 support: result.support,
368 accountId: ownerActor.Account.id
376 return videoChannelCreated
379 async function refreshActorIfNeeded (
380 actorArg: ActorModel,
381 fetchedType: ActorFetchByUrlType
382 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
383 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
385 // We need more attributes
386 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
389 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
390 const { result, statusCode } = await fetchRemoteActor(actorUrl)
392 if (statusCode === 404) {
393 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
394 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
395 return { actor: undefined, refreshed: false }
398 if (result === undefined) {
399 logger.warn('Cannot fetch remote actor in refresh actor.')
400 return { actor, refreshed: false }
403 return sequelizeTypescript.transaction(async t => {
404 updateInstanceWithAnother(actor, result.actor)
406 if (result.avatarName !== undefined) {
407 await updateActorAvatarInstance(actor, result.avatarName, t)
411 actor.setDataValue('updatedAt', new Date())
412 await actor.save({ transaction: t })
415 actor.Account.set('name', result.name)
416 actor.Account.set('description', result.summary)
418 await actor.Account.save({ transaction: t })
419 } else if (actor.VideoChannel) {
420 actor.VideoChannel.set('name', result.name)
421 actor.VideoChannel.set('description', result.summary)
422 actor.VideoChannel.set('support', result.support)
424 await actor.VideoChannel.save({ transaction: t })
427 return { refreshed: true, actor }
430 logger.warn('Cannot refresh actor.', { err })
431 return { actor, refreshed: false }