1 import * as Bluebird from 'bluebird'
2 import { join } from 'path'
3 import { Transaction } from 'sequelize'
4 import * as url from 'url'
5 import * as uuidv4 from 'uuid/v4'
6 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
7 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
8 import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
9 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
10 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
11 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
14 import { doRequest, downloadImage } from '../../helpers/requests'
15 import { getUrlFromWebfinger } from '../../helpers/webfinger'
16 import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers'
17 import { AccountModel } from '../../models/account/account'
18 import { ActorModel } from '../../models/activitypub/actor'
19 import { AvatarModel } from '../../models/avatar/avatar'
20 import { ServerModel } from '../../models/server/server'
21 import { VideoChannelModel } from '../../models/video/video-channel'
22 import { JobQueue } from '../job-queue'
23 import { getServerActor } from '../../helpers/utils'
24 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
26 // Set account keys, this could be long so process after the account creation and do not block the client
27 function setAsyncActorKeys (actor: ActorModel) {
28 return createPrivateAndPublicKeys()
29 .then(({ publicKey, privateKey }) => {
30 actor.set('publicKey', publicKey)
31 actor.set('privateKey', privateKey)
35 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
40 async function getOrCreateActorAndServerAndModel (
41 activityActor: string | ActivityPubActor,
42 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
43 recurseIfNeeded = true,
44 updateCollections = false
46 const actorUrl = getAPUrl(activityActor)
49 let actor = await fetchActorByUrl(actorUrl, fetchType)
50 // Orphan actor (not associated to an account of channel) so recreate it
51 if (actor && (!actor.Account && !actor.VideoChannel)) {
56 // We don't have this actor in our database, fetch it on remote
58 const { result } = await fetchRemoteActor(actorUrl)
59 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
61 // Create the attributed to actor
62 // In PeerTube a video channel is owned by an account
63 let ownerActor: ActorModel = undefined
64 if (recurseIfNeeded === true && result.actor.type === 'Group') {
65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
68 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
69 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
73 // Don't recurse another time
74 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
76 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
81 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
85 if (actor.Account) actor.Account.Actor = actor
86 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
88 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
89 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
91 if ((created === true || refreshed === true) && updateCollections === true) {
92 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
93 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
99 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
100 return new ActorModel({
109 inboxUrl: url + '/inbox',
110 outboxUrl: url + '/outbox',
111 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
112 followersUrl: url + '/followers',
113 followingUrl: url + '/following'
117 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
118 const followersCount = await fetchActorTotalItems(attributes.followers)
119 const followingCount = await fetchActorTotalItems(attributes.following)
121 actorInstance.set('type', attributes.type)
122 actorInstance.set('uuid', attributes.uuid)
123 actorInstance.set('preferredUsername', attributes.preferredUsername)
124 actorInstance.set('url', attributes.id)
125 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
126 actorInstance.set('followersCount', followersCount)
127 actorInstance.set('followingCount', followingCount)
128 actorInstance.set('inboxUrl', attributes.inbox)
129 actorInstance.set('outboxUrl', attributes.outbox)
130 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
131 actorInstance.set('followersUrl', attributes.followers)
132 actorInstance.set('followingUrl', attributes.following)
135 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
136 if (avatarName !== undefined) {
137 if (actorInstance.avatarId) {
139 await actorInstance.Avatar.destroy({ transaction: t })
141 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
145 const avatar = await AvatarModel.create({
147 }, { transaction: t })
149 actorInstance.set('avatarId', avatar.id)
150 actorInstance.Avatar = avatar
156 async function fetchActorTotalItems (url: string) {
165 const { body } = await doRequest(options)
166 return body.totalItems ? body.totalItems : 0
168 logger.warn('Cannot fetch remote actor count %s.', url, { err })
173 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
175 actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
176 isActivityPubUrlValid(actorJSON.icon.url)
178 const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
180 const avatarName = uuidv4() + extension
181 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
189 async function addFetchOutboxJob (actor: ActorModel) {
190 // Don't fetch ourselves
191 const serverActor = await getServerActor()
192 if (serverActor.id === actor.id) {
193 logger.error('Cannot fetch our own outbox!')
198 uri: actor.outboxUrl,
199 type: 'activity' as 'activity'
202 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
206 getOrCreateActorAndServerAndModel,
209 fetchActorTotalItems,
212 updateActorAvatarInstance,
216 // ---------------------------------------------------------------------------
218 function saveActorAndServerAndModelIfNotExist (
219 result: FetchRemoteActorResult,
220 ownerActor?: ActorModel,
222 ): Bluebird<ActorModel> | Promise<ActorModel> {
223 let actor = result.actor
225 if (t !== undefined) return save(t)
227 return sequelizeTypescript.transaction(t => save(t))
229 async function save (t: Transaction) {
230 const actorHost = url.parse(actor.url).host
232 const serverOptions = {
241 const [ server ] = await ServerModel.findOrCreate(serverOptions)
243 // Save our new account in database
244 actor.set('serverId', server.id)
247 if (result.avatarName) {
248 const avatar = await AvatarModel.create({
249 filename: result.avatarName
250 }, { transaction: t })
251 actor.set('avatarId', avatar.id)
254 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
255 // (which could be false in a retried query)
256 const [ actorCreated ] = await ActorModel.findOrCreate({
257 defaults: actor.toJSON(),
264 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
265 actorCreated.Account = await saveAccount(actorCreated, result, t)
266 actorCreated.Account.Actor = actorCreated
267 } else if (actorCreated.type === 'Group') { // Video channel
268 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
269 actorCreated.VideoChannel.Actor = actorCreated
270 actorCreated.VideoChannel.Account = ownerActor.Account
277 type FetchRemoteActorResult = {
283 attributedTo: ActivityPubAttributedTo[]
285 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
293 logger.info('Fetching remote actor %s.', actorUrl)
295 const requestResult = await doRequest(options)
296 normalizeActor(requestResult.body)
298 const actorJSON: ActivityPubActor = requestResult.body
299 if (isActorObjectValid(actorJSON) === false) {
300 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
301 return { result: undefined, statusCode: requestResult.response.statusCode }
304 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
305 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
308 const followersCount = await fetchActorTotalItems(actorJSON.followers)
309 const followingCount = await fetchActorTotalItems(actorJSON.following)
311 const actor = new ActorModel({
312 type: actorJSON.type,
313 uuid: actorJSON.uuid,
314 preferredUsername: actorJSON.preferredUsername,
316 publicKey: actorJSON.publicKey.publicKeyPem,
318 followersCount: followersCount,
319 followingCount: followingCount,
320 inboxUrl: actorJSON.inbox,
321 outboxUrl: actorJSON.outbox,
322 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
323 followersUrl: actorJSON.followers,
324 followingUrl: actorJSON.following
327 const avatarName = await fetchAvatarIfExists(actorJSON)
329 const name = actorJSON.name || actorJSON.preferredUsername
331 statusCode: requestResult.response.statusCode,
336 summary: actorJSON.summary,
337 support: actorJSON.support,
338 attributedTo: actorJSON.attributedTo
343 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
344 const [ accountCreated ] = await AccountModel.findOrCreate({
347 description: result.summary,
356 return accountCreated
359 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
360 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
363 description: result.summary,
364 support: result.support,
366 accountId: ownerActor.Account.id
374 return videoChannelCreated
377 async function refreshActorIfNeeded (
378 actorArg: ActorModel,
379 fetchedType: ActorFetchByUrlType
380 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
381 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
383 // We need more attributes
384 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
387 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
388 const { result, statusCode } = await fetchRemoteActor(actorUrl)
390 if (statusCode === 404) {
391 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
392 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
393 return { actor: undefined, refreshed: false }
396 if (result === undefined) {
397 logger.warn('Cannot fetch remote actor in refresh actor.')
398 return { actor, refreshed: false }
401 return sequelizeTypescript.transaction(async t => {
402 updateInstanceWithAnother(actor, result.actor)
404 if (result.avatarName !== undefined) {
405 await updateActorAvatarInstance(actor, result.avatarName, t)
409 actor.setDataValue('updatedAt', new Date())
410 await actor.save({ transaction: t })
413 actor.Account.set('name', result.name)
414 actor.Account.set('description', result.summary)
416 await actor.Account.save({ transaction: t })
417 } else if (actor.VideoChannel) {
418 actor.VideoChannel.set('name', result.name)
419 actor.VideoChannel.set('description', result.summary)
420 actor.VideoChannel.set('support', result.support)
422 await actor.VideoChannel.save({ transaction: t })
425 return { refreshed: true, actor }
428 logger.warn('Cannot refresh actor.', { err })
429 return { actor, refreshed: false }