1 import * as Bluebird from 'bluebird'
2 import { join } from 'path'
3 import { Transaction } from 'sequelize'
4 import * as url from 'url'
5 import * as uuidv4 from 'uuid/v4'
6 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
7 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
8 import { getActorUrl } from '../../helpers/activitypub'
9 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
10 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
11 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
12 import { logger } from '../../helpers/logger'
13 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
14 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
15 import { getUrlFromWebfinger } from '../../helpers/webfinger'
16 import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers'
17 import { AccountModel } from '../../models/account/account'
18 import { ActorModel } from '../../models/activitypub/actor'
19 import { AvatarModel } from '../../models/avatar/avatar'
20 import { ServerModel } from '../../models/server/server'
21 import { VideoChannelModel } from '../../models/video/video-channel'
22 import { JobQueue } from '../job-queue'
23 import { getServerActor } from '../../helpers/utils'
24 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
26 // Set account keys, this could be long so process after the account creation and do not block the client
27 function setAsyncActorKeys (actor: ActorModel) {
28 return createPrivateAndPublicKeys()
29 .then(({ publicKey, privateKey }) => {
30 actor.set('publicKey', publicKey)
31 actor.set('privateKey', privateKey)
35 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
40 async function getOrCreateActorAndServerAndModel (
41 activityActor: string | ActivityPubActor,
42 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
43 recurseIfNeeded = true,
44 updateCollections = false
46 const actorUrl = getActorUrl(activityActor)
49 let actor = await fetchActorByUrl(actorUrl, fetchType)
50 // Orphan actor (not associated to an account of channel) so recreate it
51 if (actor && (!actor.Account && !actor.VideoChannel)) {
56 // We don't have this actor in our database, fetch it on remote
58 const { result } = await fetchRemoteActor(actorUrl)
59 if (result === undefined) throw new Error('Cannot fetch remote actor.')
61 // Create the attributed to actor
62 // In PeerTube a video channel is owned by an account
63 let ownerActor: ActorModel = undefined
64 if (recurseIfNeeded === true && result.actor.type === 'Group') {
65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
69 // Assert we don't recurse another time
70 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
72 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
77 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
81 if (actor.Account) actor.Account.Actor = actor
82 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
84 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
85 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
87 if ((created === true || refreshed === true) && updateCollections === true) {
88 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
89 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
95 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
96 return new ActorModel({
105 inboxUrl: url + '/inbox',
106 outboxUrl: url + '/outbox',
107 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
108 followersUrl: url + '/followers',
109 followingUrl: url + '/following'
113 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
114 const followersCount = await fetchActorTotalItems(attributes.followers)
115 const followingCount = await fetchActorTotalItems(attributes.following)
117 actorInstance.set('type', attributes.type)
118 actorInstance.set('uuid', attributes.uuid)
119 actorInstance.set('preferredUsername', attributes.preferredUsername)
120 actorInstance.set('url', attributes.id)
121 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
122 actorInstance.set('followersCount', followersCount)
123 actorInstance.set('followingCount', followingCount)
124 actorInstance.set('inboxUrl', attributes.inbox)
125 actorInstance.set('outboxUrl', attributes.outbox)
126 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
127 actorInstance.set('followersUrl', attributes.followers)
128 actorInstance.set('followingUrl', attributes.following)
131 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
132 if (avatarName !== undefined) {
133 if (actorInstance.avatarId) {
135 await actorInstance.Avatar.destroy({ transaction: t })
137 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
141 const avatar = await AvatarModel.create({
143 }, { transaction: t })
145 actorInstance.set('avatarId', avatar.id)
146 actorInstance.Avatar = avatar
152 async function fetchActorTotalItems (url: string) {
161 const { body } = await doRequest(options)
162 return body.totalItems ? body.totalItems : 0
164 logger.warn('Cannot fetch remote actor count %s.', url, { err })
169 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
171 actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
172 isActivityPubUrlValid(actorJSON.icon.url)
174 const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
176 const avatarName = uuidv4() + extension
177 const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
179 await doRequestAndSaveToFile({
181 uri: actorJSON.icon.url
190 async function addFetchOutboxJob (actor: ActorModel) {
191 // Don't fetch ourselves
192 const serverActor = await getServerActor()
193 if (serverActor.id === actor.id) {
194 logger.error('Cannot fetch our own outbox!')
199 uri: actor.outboxUrl,
200 type: 'activity' as 'activity'
203 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
207 getOrCreateActorAndServerAndModel,
210 fetchActorTotalItems,
213 updateActorAvatarInstance,
217 // ---------------------------------------------------------------------------
219 function saveActorAndServerAndModelIfNotExist (
220 result: FetchRemoteActorResult,
221 ownerActor?: ActorModel,
223 ): Bluebird<ActorModel> | Promise<ActorModel> {
224 let actor = result.actor
226 if (t !== undefined) return save(t)
228 return sequelizeTypescript.transaction(t => save(t))
230 async function save (t: Transaction) {
231 const actorHost = url.parse(actor.url).host
233 const serverOptions = {
242 const [ server ] = await ServerModel.findOrCreate(serverOptions)
244 // Save our new account in database
245 actor.set('serverId', server.id)
248 if (result.avatarName) {
249 const avatar = await AvatarModel.create({
250 filename: result.avatarName
251 }, { transaction: t })
252 actor.set('avatarId', avatar.id)
255 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
256 // (which could be false in a retried query)
257 const [ actorCreated ] = await ActorModel.findOrCreate({
258 defaults: actor.toJSON(),
265 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
266 actorCreated.Account = await saveAccount(actorCreated, result, t)
267 actorCreated.Account.Actor = actorCreated
268 } else if (actorCreated.type === 'Group') { // Video channel
269 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
270 actorCreated.VideoChannel.Actor = actorCreated
271 actorCreated.VideoChannel.Account = ownerActor.Account
278 type FetchRemoteActorResult = {
284 attributedTo: ActivityPubAttributedTo[]
286 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
294 logger.info('Fetching remote actor %s.', actorUrl)
296 const requestResult = await doRequest(options)
297 normalizeActor(requestResult.body)
299 const actorJSON: ActivityPubActor = requestResult.body
301 if (isActorObjectValid(actorJSON) === false) {
302 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
303 return { result: undefined, statusCode: requestResult.response.statusCode }
306 const followersCount = await fetchActorTotalItems(actorJSON.followers)
307 const followingCount = await fetchActorTotalItems(actorJSON.following)
309 const actor = new ActorModel({
310 type: actorJSON.type,
311 uuid: actorJSON.uuid,
312 preferredUsername: actorJSON.preferredUsername,
314 publicKey: actorJSON.publicKey.publicKeyPem,
316 followersCount: followersCount,
317 followingCount: followingCount,
318 inboxUrl: actorJSON.inbox,
319 outboxUrl: actorJSON.outbox,
320 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
321 followersUrl: actorJSON.followers,
322 followingUrl: actorJSON.following
325 const avatarName = await fetchAvatarIfExists(actorJSON)
327 const name = actorJSON.name || actorJSON.preferredUsername
329 statusCode: requestResult.response.statusCode,
334 summary: actorJSON.summary,
335 support: actorJSON.support,
336 attributedTo: actorJSON.attributedTo
341 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
342 const [ accountCreated ] = await AccountModel.findOrCreate({
345 description: result.summary,
354 return accountCreated
357 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
358 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
361 description: result.summary,
362 support: result.support,
364 accountId: ownerActor.Account.id
372 return videoChannelCreated
375 async function refreshActorIfNeeded (
376 actorArg: ActorModel,
377 fetchedType: ActorFetchByUrlType
378 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
379 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
381 // We need more attributes
382 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
385 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
386 const { result, statusCode } = await fetchRemoteActor(actorUrl)
388 if (statusCode === 404) {
389 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
390 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
391 return { actor: undefined, refreshed: false }
394 if (result === undefined) {
395 logger.warn('Cannot fetch remote actor in refresh actor.')
396 return { actor, refreshed: false }
399 return sequelizeTypescript.transaction(async t => {
400 updateInstanceWithAnother(actor, result.actor)
402 if (result.avatarName !== undefined) {
403 await updateActorAvatarInstance(actor, result.avatarName, t)
407 actor.setDataValue('updatedAt', new Date())
408 await actor.save({ transaction: t })
411 actor.Account.set('name', result.name)
412 actor.Account.set('description', result.summary)
414 await actor.Account.save({ transaction: t })
415 } else if (actor.VideoChannel) {
416 actor.VideoChannel.set('name', result.name)
417 actor.VideoChannel.set('description', result.summary)
418 actor.VideoChannel.set('support', result.support)
420 await actor.VideoChannel.save({ transaction: t })
423 return { refreshed: true, actor }
426 logger.warn('Cannot refresh actor.', { err })
427 return { actor, refreshed: false }