1 import * as Bluebird from 'bluebird'
2 import { Transaction } from 'sequelize'
3 import * as url from 'url'
4 import * as uuidv4 from 'uuid/v4'
5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11 import { logger } from '../../helpers/logger'
12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13 import { doRequest, downloadImage } from '../../helpers/requests'
14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
16 import { AccountModel } from '../../models/account/account'
17 import { ActorModel } from '../../models/activitypub/actor'
18 import { AvatarModel } from '../../models/avatar/avatar'
19 import { ServerModel } from '../../models/server/server'
20 import { VideoChannelModel } from '../../models/video/video-channel'
21 import { JobQueue } from '../job-queue'
22 import { getServerActor } from '../../helpers/utils'
23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
25 // Set account keys, this could be long so process after the account creation and do not block the client
26 function setAsyncActorKeys (actor: ActorModel) {
27 return createPrivateAndPublicKeys()
28 .then(({ publicKey, privateKey }) => {
29 actor.set('publicKey', publicKey)
30 actor.set('privateKey', privateKey)
34 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
39 async function getOrCreateActorAndServerAndModel (
40 activityActor: string | ActivityPubActor,
41 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
42 recurseIfNeeded = true,
43 updateCollections = false
45 const actorUrl = getAPId(activityActor)
47 let accountPlaylistsUrl: string
49 let actor = await fetchActorByUrl(actorUrl, fetchType)
50 // Orphan actor (not associated to an account of channel) so recreate it
51 if (actor && (!actor.Account && !actor.VideoChannel)) {
56 // We don't have this actor in our database, fetch it on remote
58 const { result } = await fetchRemoteActor(actorUrl)
59 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
61 // Create the attributed to actor
62 // In PeerTube a video channel is owned by an account
63 let ownerActor: ActorModel = undefined
64 if (recurseIfNeeded === true && result.actor.type === 'Group') {
65 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
66 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
68 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
69 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
73 // Don't recurse another time
74 const recurseIfNeeded = false
75 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
77 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
82 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
84 accountPlaylistsUrl = result.playlists
87 if (actor.Account) actor.Account.Actor = actor
88 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
90 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
91 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
93 if ((created === true || refreshed === true) && updateCollections === true) {
94 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
95 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
98 // We created a new account: fetch the playlists
99 if (created === true && actor.Account && accountPlaylistsUrl) {
100 const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
101 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
104 return actorRefreshed
107 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
108 return new ActorModel({
117 inboxUrl: url + '/inbox',
118 outboxUrl: url + '/outbox',
119 sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
120 followersUrl: url + '/followers',
121 followingUrl: url + '/following'
125 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
126 const followersCount = await fetchActorTotalItems(attributes.followers)
127 const followingCount = await fetchActorTotalItems(attributes.following)
129 actorInstance.set('type', attributes.type)
130 actorInstance.set('uuid', attributes.uuid)
131 actorInstance.set('preferredUsername', attributes.preferredUsername)
132 actorInstance.set('url', attributes.id)
133 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
134 actorInstance.set('followersCount', followersCount)
135 actorInstance.set('followingCount', followingCount)
136 actorInstance.set('inboxUrl', attributes.inbox)
137 actorInstance.set('outboxUrl', attributes.outbox)
138 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
139 actorInstance.set('followersUrl', attributes.followers)
140 actorInstance.set('followingUrl', attributes.following)
143 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
144 if (avatarName !== undefined) {
145 if (actorInstance.avatarId) {
147 await actorInstance.Avatar.destroy({ transaction: t })
149 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
153 const avatar = await AvatarModel.create({
155 }, { transaction: t })
157 actorInstance.set('avatarId', avatar.id)
158 actorInstance.Avatar = avatar
164 async function fetchActorTotalItems (url: string) {
173 const { body } = await doRequest(options)
174 return body.totalItems ? body.totalItems : 0
176 logger.warn('Cannot fetch remote actor count %s.', url, { err })
181 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
183 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
184 isActivityPubUrlValid(actorJSON.icon.url)
186 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
188 const avatarName = uuidv4() + extension
189 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
197 async function addFetchOutboxJob (actor: ActorModel) {
198 // Don't fetch ourselves
199 const serverActor = await getServerActor()
200 if (serverActor.id === actor.id) {
201 logger.error('Cannot fetch our own outbox!')
206 uri: actor.outboxUrl,
207 type: 'activity' as 'activity'
210 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
213 async function refreshActorIfNeeded (
214 actorArg: ActorModel,
215 fetchedType: ActorFetchByUrlType
216 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
217 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
219 // We need more attributes
220 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
225 actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
227 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
231 const { result, statusCode } = await fetchRemoteActor(actorUrl)
233 if (statusCode === 404) {
234 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
235 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
236 return { actor: undefined, refreshed: false }
239 if (result === undefined) {
240 logger.warn('Cannot fetch remote actor in refresh actor.')
241 return { actor, refreshed: false }
244 return sequelizeTypescript.transaction(async t => {
245 updateInstanceWithAnother(actor, result.actor)
247 if (result.avatarName !== undefined) {
248 await updateActorAvatarInstance(actor, result.avatarName, t)
252 actor.setDataValue('updatedAt', new Date())
253 await actor.save({ transaction: t })
256 actor.Account.set('name', result.name)
257 actor.Account.set('description', result.summary)
259 await actor.Account.save({ transaction: t })
260 } else if (actor.VideoChannel) {
261 actor.VideoChannel.set('name', result.name)
262 actor.VideoChannel.set('description', result.summary)
263 actor.VideoChannel.set('support', result.support)
265 await actor.VideoChannel.save({ transaction: t })
268 return { refreshed: true, actor }
271 logger.warn('Cannot refresh actor.', { err })
272 return { actor, refreshed: false }
277 getOrCreateActorAndServerAndModel,
280 fetchActorTotalItems,
283 refreshActorIfNeeded,
284 updateActorAvatarInstance,
288 // ---------------------------------------------------------------------------
290 function saveActorAndServerAndModelIfNotExist (
291 result: FetchRemoteActorResult,
292 ownerActor?: ActorModel,
294 ): Bluebird<ActorModel> | Promise<ActorModel> {
295 let actor = result.actor
297 if (t !== undefined) return save(t)
299 return sequelizeTypescript.transaction(t => save(t))
301 async function save (t: Transaction) {
302 const actorHost = url.parse(actor.url).host
304 const serverOptions = {
313 const [ server ] = await ServerModel.findOrCreate(serverOptions)
315 // Save our new account in database
316 actor.set('serverId', server.id)
319 if (result.avatarName) {
320 const avatar = await AvatarModel.create({
321 filename: result.avatarName
322 }, { transaction: t })
323 actor.set('avatarId', avatar.id)
326 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
327 // (which could be false in a retried query)
328 const [ actorCreated ] = await ActorModel.findOrCreate({
329 defaults: actor.toJSON(),
336 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
337 actorCreated.Account = await saveAccount(actorCreated, result, t)
338 actorCreated.Account.Actor = actorCreated
339 } else if (actorCreated.type === 'Group') { // Video channel
340 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
341 actorCreated.VideoChannel.Actor = actorCreated
342 actorCreated.VideoChannel.Account = ownerActor.Account
349 type FetchRemoteActorResult = {
356 attributedTo: ActivityPubAttributedTo[]
358 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
366 logger.info('Fetching remote actor %s.', actorUrl)
368 const requestResult = await doRequest<ActivityPubActor>(options)
369 normalizeActor(requestResult.body)
371 const actorJSON = requestResult.body
372 if (isActorObjectValid(actorJSON) === false) {
373 logger.debug('Remote actor JSON is not valid.', { actorJSON })
374 return { result: undefined, statusCode: requestResult.response.statusCode }
377 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
378 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
379 return { result: undefined, statusCode: requestResult.response.statusCode }
382 const followersCount = await fetchActorTotalItems(actorJSON.followers)
383 const followingCount = await fetchActorTotalItems(actorJSON.following)
385 const actor = new ActorModel({
386 type: actorJSON.type,
387 uuid: actorJSON.uuid,
388 preferredUsername: actorJSON.preferredUsername,
390 publicKey: actorJSON.publicKey.publicKeyPem,
392 followersCount: followersCount,
393 followingCount: followingCount,
394 inboxUrl: actorJSON.inbox,
395 outboxUrl: actorJSON.outbox,
396 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
397 followersUrl: actorJSON.followers,
398 followingUrl: actorJSON.following
401 const avatarName = await fetchAvatarIfExists(actorJSON)
403 const name = actorJSON.name || actorJSON.preferredUsername
405 statusCode: requestResult.response.statusCode,
410 summary: actorJSON.summary,
411 support: actorJSON.support,
412 playlists: actorJSON.playlists,
413 attributedTo: actorJSON.attributedTo
418 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
419 const [ accountCreated ] = await AccountModel.findOrCreate({
422 description: result.summary,
431 return accountCreated
434 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
435 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
438 description: result.summary,
439 support: result.support,
441 accountId: ownerActor.Account.id
449 return videoChannelCreated