1 import * as Bluebird from 'bluebird'
2 import { Transaction } from 'sequelize'
3 import * as url from 'url'
4 import * as uuidv4 from 'uuid/v4'
5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
7 import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11 import { logger } from '../../helpers/logger'
12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13 import { doRequest, downloadImage } from '../../helpers/requests'
14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
16 import { AccountModel } from '../../models/account/account'
17 import { ActorModel } from '../../models/activitypub/actor'
18 import { AvatarModel } from '../../models/avatar/avatar'
19 import { ServerModel } from '../../models/server/server'
20 import { VideoChannelModel } from '../../models/video/video-channel'
21 import { JobQueue } from '../job-queue'
22 import { getServerActor } from '../../helpers/utils'
23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
25 // Set account keys, this could be long so process after the account creation and do not block the client
26 function setAsyncActorKeys (actor: ActorModel) {
27 return createPrivateAndPublicKeys()
28 .then(({ publicKey, privateKey }) => {
29 actor.set('publicKey', publicKey)
30 actor.set('privateKey', privateKey)
34 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
39 async function getOrCreateActorAndServerAndModel (
40 activityActor: string | ActivityPubActor,
41 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
42 recurseIfNeeded = true,
43 updateCollections = false
45 const actorUrl = getAPUrl(activityActor)
48 let actor = await fetchActorByUrl(actorUrl, fetchType)
49 // Orphan actor (not associated to an account of channel) so recreate it
50 if (actor && (!actor.Account && !actor.VideoChannel)) {
55 // We don't have this actor in our database, fetch it on remote
57 const { result } = await fetchRemoteActor(actorUrl)
58 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
60 // Create the attributed to actor
61 // In PeerTube a video channel is owned by an account
62 let ownerActor: ActorModel = undefined
63 if (recurseIfNeeded === true && result.actor.type === 'Group') {
64 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
65 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
67 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
68 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
72 // Don't recurse another time
73 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
75 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
80 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
84 if (actor.Account) actor.Account.Actor = actor
85 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
87 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
88 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
90 if ((created === true || refreshed === true) && updateCollections === true) {
91 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
92 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
98 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
99 return new ActorModel({
108 inboxUrl: url + '/inbox',
109 outboxUrl: url + '/outbox',
110 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
111 followersUrl: url + '/followers',
112 followingUrl: url + '/following'
116 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
117 const followersCount = await fetchActorTotalItems(attributes.followers)
118 const followingCount = await fetchActorTotalItems(attributes.following)
120 actorInstance.set('type', attributes.type)
121 actorInstance.set('uuid', attributes.uuid)
122 actorInstance.set('preferredUsername', attributes.preferredUsername)
123 actorInstance.set('url', attributes.id)
124 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
125 actorInstance.set('followersCount', followersCount)
126 actorInstance.set('followingCount', followingCount)
127 actorInstance.set('inboxUrl', attributes.inbox)
128 actorInstance.set('outboxUrl', attributes.outbox)
129 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
130 actorInstance.set('followersUrl', attributes.followers)
131 actorInstance.set('followingUrl', attributes.following)
134 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
135 if (avatarName !== undefined) {
136 if (actorInstance.avatarId) {
138 await actorInstance.Avatar.destroy({ transaction: t })
140 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
144 const avatar = await AvatarModel.create({
146 }, { transaction: t })
148 actorInstance.set('avatarId', avatar.id)
149 actorInstance.Avatar = avatar
155 async function fetchActorTotalItems (url: string) {
164 const { body } = await doRequest(options)
165 return body.totalItems ? body.totalItems : 0
167 logger.warn('Cannot fetch remote actor count %s.', url, { err })
172 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
174 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
175 isActivityPubUrlValid(actorJSON.icon.url)
177 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
179 const avatarName = uuidv4() + extension
180 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
188 async function addFetchOutboxJob (actor: ActorModel) {
189 // Don't fetch ourselves
190 const serverActor = await getServerActor()
191 if (serverActor.id === actor.id) {
192 logger.error('Cannot fetch our own outbox!')
197 uri: actor.outboxUrl,
198 type: 'activity' as 'activity'
201 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
205 getOrCreateActorAndServerAndModel,
208 fetchActorTotalItems,
211 updateActorAvatarInstance,
215 // ---------------------------------------------------------------------------
217 function saveActorAndServerAndModelIfNotExist (
218 result: FetchRemoteActorResult,
219 ownerActor?: ActorModel,
221 ): Bluebird<ActorModel> | Promise<ActorModel> {
222 let actor = result.actor
224 if (t !== undefined) return save(t)
226 return sequelizeTypescript.transaction(t => save(t))
228 async function save (t: Transaction) {
229 const actorHost = url.parse(actor.url).host
231 const serverOptions = {
240 const [ server ] = await ServerModel.findOrCreate(serverOptions)
242 // Save our new account in database
243 actor.set('serverId', server.id)
246 if (result.avatarName) {
247 const avatar = await AvatarModel.create({
248 filename: result.avatarName
249 }, { transaction: t })
250 actor.set('avatarId', avatar.id)
253 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
254 // (which could be false in a retried query)
255 const [ actorCreated ] = await ActorModel.findOrCreate({
256 defaults: actor.toJSON(),
263 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
264 actorCreated.Account = await saveAccount(actorCreated, result, t)
265 actorCreated.Account.Actor = actorCreated
266 } else if (actorCreated.type === 'Group') { // Video channel
267 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
268 actorCreated.VideoChannel.Actor = actorCreated
269 actorCreated.VideoChannel.Account = ownerActor.Account
276 type FetchRemoteActorResult = {
282 attributedTo: ActivityPubAttributedTo[]
284 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
292 logger.info('Fetching remote actor %s.', actorUrl)
294 const requestResult = await doRequest(options)
295 normalizeActor(requestResult.body)
297 const actorJSON: ActivityPubActor = requestResult.body
298 if (isActorObjectValid(actorJSON) === false) {
299 logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
300 return { result: undefined, statusCode: requestResult.response.statusCode }
303 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
304 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
307 const followersCount = await fetchActorTotalItems(actorJSON.followers)
308 const followingCount = await fetchActorTotalItems(actorJSON.following)
310 const actor = new ActorModel({
311 type: actorJSON.type,
312 uuid: actorJSON.uuid,
313 preferredUsername: actorJSON.preferredUsername,
315 publicKey: actorJSON.publicKey.publicKeyPem,
317 followersCount: followersCount,
318 followingCount: followingCount,
319 inboxUrl: actorJSON.inbox,
320 outboxUrl: actorJSON.outbox,
321 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
322 followersUrl: actorJSON.followers,
323 followingUrl: actorJSON.following
326 const avatarName = await fetchAvatarIfExists(actorJSON)
328 const name = actorJSON.name || actorJSON.preferredUsername
330 statusCode: requestResult.response.statusCode,
335 summary: actorJSON.summary,
336 support: actorJSON.support,
337 attributedTo: actorJSON.attributedTo
342 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
343 const [ accountCreated ] = await AccountModel.findOrCreate({
346 description: result.summary,
355 return accountCreated
358 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
359 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
362 description: result.summary,
363 support: result.support,
365 accountId: ownerActor.Account.id
373 return videoChannelCreated
376 async function refreshActorIfNeeded (
377 actorArg: ActorModel,
378 fetchedType: ActorFetchByUrlType
379 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
380 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
382 // We need more attributes
383 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
386 const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
387 const { result, statusCode } = await fetchRemoteActor(actorUrl)
389 if (statusCode === 404) {
390 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
391 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
392 return { actor: undefined, refreshed: false }
395 if (result === undefined) {
396 logger.warn('Cannot fetch remote actor in refresh actor.')
397 return { actor, refreshed: false }
400 return sequelizeTypescript.transaction(async t => {
401 updateInstanceWithAnother(actor, result.actor)
403 if (result.avatarName !== undefined) {
404 await updateActorAvatarInstance(actor, result.avatarName, t)
408 actor.setDataValue('updatedAt', new Date())
409 await actor.save({ transaction: t })
412 actor.Account.set('name', result.name)
413 actor.Account.set('description', result.summary)
415 await actor.Account.save({ transaction: t })
416 } else if (actor.VideoChannel) {
417 actor.VideoChannel.set('name', result.name)
418 actor.VideoChannel.set('description', result.summary)
419 actor.VideoChannel.set('support', result.support)
421 await actor.VideoChannel.save({ transaction: t })
424 return { refreshed: true, actor }
427 logger.warn('Cannot refresh actor.', { err })
428 return { actor, refreshed: false }