1 import * as Bluebird from 'bluebird'
2 import { Transaction } from 'sequelize'
3 import * as url from 'url'
4 import * as uuidv4 from 'uuid/v4'
5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
11 import { logger } from '../../helpers/logger'
12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
13 import { doRequest, downloadImage } from '../../helpers/requests'
14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
15 import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants'
16 import { AccountModel } from '../../models/account/account'
17 import { ActorModel } from '../../models/activitypub/actor'
18 import { AvatarModel } from '../../models/avatar/avatar'
19 import { ServerModel } from '../../models/server/server'
20 import { VideoChannelModel } from '../../models/video/video-channel'
21 import { JobQueue } from '../job-queue'
22 import { getServerActor } from '../../helpers/utils'
23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
24 import { CONFIG } from '../../initializers/config'
25 import { sequelizeTypescript } from '../../initializers/database'
27 // Set account keys, this could be long so process after the account creation and do not block the client
28 function setAsyncActorKeys (actor: ActorModel) {
29 return createPrivateAndPublicKeys()
30 .then(({ publicKey, privateKey }) => {
31 actor.set('publicKey', publicKey)
32 actor.set('privateKey', privateKey)
36 logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
41 async function getOrCreateActorAndServerAndModel (
42 activityActor: string | ActivityPubActor,
43 fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
44 recurseIfNeeded = true,
45 updateCollections = false
47 const actorUrl = getAPId(activityActor)
49 let accountPlaylistsUrl: string
51 let actor = await fetchActorByUrl(actorUrl, fetchType)
52 // Orphan actor (not associated to an account of channel) so recreate it
53 if (actor && (!actor.Account && !actor.VideoChannel)) {
58 // We don't have this actor in our database, fetch it on remote
60 const { result } = await fetchRemoteActor(actorUrl)
61 if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
63 // Create the attributed to actor
64 // In PeerTube a video channel is owned by an account
65 let ownerActor: ActorModel = undefined
66 if (recurseIfNeeded === true && result.actor.type === 'Group') {
67 const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
68 if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
70 if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
71 throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
75 // Don't recurse another time
76 const recurseIfNeeded = false
77 ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
79 logger.error('Cannot get or create account attributed to video channel ' + actor.url)
84 actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
86 accountPlaylistsUrl = result.playlists
89 if (actor.Account) actor.Account.Actor = actor
90 if (actor.VideoChannel) actor.VideoChannel.Actor = actor
92 const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
93 if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
95 if ((created === true || refreshed === true) && updateCollections === true) {
96 const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
97 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
100 // We created a new account: fetch the playlists
101 if (created === true && actor.Account && accountPlaylistsUrl) {
102 const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
103 await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
106 return actorRefreshed
109 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
110 return new ActorModel({
119 inboxUrl: url + '/inbox',
120 outboxUrl: url + '/outbox',
121 sharedInboxUrl: WEBSERVER.URL + '/inbox',
122 followersUrl: url + '/followers',
123 followingUrl: url + '/following'
127 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
128 const followersCount = await fetchActorTotalItems(attributes.followers)
129 const followingCount = await fetchActorTotalItems(attributes.following)
131 actorInstance.set('type', attributes.type)
132 actorInstance.set('uuid', attributes.uuid)
133 actorInstance.set('preferredUsername', attributes.preferredUsername)
134 actorInstance.set('url', attributes.id)
135 actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
136 actorInstance.set('followersCount', followersCount)
137 actorInstance.set('followingCount', followingCount)
138 actorInstance.set('inboxUrl', attributes.inbox)
139 actorInstance.set('outboxUrl', attributes.outbox)
140 actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
141 actorInstance.set('followersUrl', attributes.followers)
142 actorInstance.set('followingUrl', attributes.following)
145 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
146 if (avatarName !== undefined) {
147 if (actorInstance.avatarId) {
149 await actorInstance.Avatar.destroy({ transaction: t })
151 logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
155 const avatar = await AvatarModel.create({
157 }, { transaction: t })
159 actorInstance.set('avatarId', avatar.id)
160 actorInstance.Avatar = avatar
166 async function fetchActorTotalItems (url: string) {
175 const { body } = await doRequest(options)
176 return body.totalItems ? body.totalItems : 0
178 logger.warn('Cannot fetch remote actor count %s.', url, { err })
183 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
185 actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
186 isActivityPubUrlValid(actorJSON.icon.url)
188 const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
190 const avatarName = uuidv4() + extension
191 await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
199 async function addFetchOutboxJob (actor: ActorModel) {
200 // Don't fetch ourselves
201 const serverActor = await getServerActor()
202 if (serverActor.id === actor.id) {
203 logger.error('Cannot fetch our own outbox!')
208 uri: actor.outboxUrl,
209 type: 'activity' as 'activity'
212 return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
215 async function refreshActorIfNeeded (
216 actorArg: ActorModel,
217 fetchedType: ActorFetchByUrlType
218 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
219 if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
221 // We need more attributes
222 const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
227 actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
229 logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
233 const { result, statusCode } = await fetchRemoteActor(actorUrl)
235 if (statusCode === 404) {
236 logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
237 actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
238 return { actor: undefined, refreshed: false }
241 if (result === undefined) {
242 logger.warn('Cannot fetch remote actor in refresh actor.')
243 return { actor, refreshed: false }
246 return sequelizeTypescript.transaction(async t => {
247 updateInstanceWithAnother(actor, result.actor)
249 if (result.avatarName !== undefined) {
250 await updateActorAvatarInstance(actor, result.avatarName, t)
254 actor.setDataValue('updatedAt', new Date())
255 await actor.save({ transaction: t })
258 actor.Account.set('name', result.name)
259 actor.Account.set('description', result.summary)
261 await actor.Account.save({ transaction: t })
262 } else if (actor.VideoChannel) {
263 actor.VideoChannel.set('name', result.name)
264 actor.VideoChannel.set('description', result.summary)
265 actor.VideoChannel.set('support', result.support)
267 await actor.VideoChannel.save({ transaction: t })
270 return { refreshed: true, actor }
273 logger.warn('Cannot refresh actor %s.', actor.url, { err })
274 return { actor, refreshed: false }
279 getOrCreateActorAndServerAndModel,
282 fetchActorTotalItems,
285 refreshActorIfNeeded,
286 updateActorAvatarInstance,
290 // ---------------------------------------------------------------------------
292 function saveActorAndServerAndModelIfNotExist (
293 result: FetchRemoteActorResult,
294 ownerActor?: ActorModel,
296 ): Bluebird<ActorModel> | Promise<ActorModel> {
297 let actor = result.actor
299 if (t !== undefined) return save(t)
301 return sequelizeTypescript.transaction(t => save(t))
303 async function save (t: Transaction) {
304 const actorHost = url.parse(actor.url).host
306 const serverOptions = {
315 const [ server ] = await ServerModel.findOrCreate(serverOptions)
317 // Save our new account in database
318 actor.set('serverId', server.id)
321 if (result.avatarName) {
322 const avatar = await AvatarModel.create({
323 filename: result.avatarName
324 }, { transaction: t })
325 actor.set('avatarId', avatar.id)
328 // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
329 // (which could be false in a retried query)
330 const [ actorCreated ] = await ActorModel.findOrCreate({
331 defaults: actor.toJSON(),
338 if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
339 actorCreated.Account = await saveAccount(actorCreated, result, t)
340 actorCreated.Account.Actor = actorCreated
341 } else if (actorCreated.type === 'Group') { // Video channel
342 actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
343 actorCreated.VideoChannel.Actor = actorCreated
344 actorCreated.VideoChannel.Account = ownerActor.Account
347 actorCreated.Server = server
353 type FetchRemoteActorResult = {
360 attributedTo: ActivityPubAttributedTo[]
362 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
370 logger.info('Fetching remote actor %s.', actorUrl)
372 const requestResult = await doRequest<ActivityPubActor>(options)
373 normalizeActor(requestResult.body)
375 const actorJSON = requestResult.body
376 if (isActorObjectValid(actorJSON) === false) {
377 logger.debug('Remote actor JSON is not valid.', { actorJSON })
378 return { result: undefined, statusCode: requestResult.response.statusCode }
381 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
382 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
383 return { result: undefined, statusCode: requestResult.response.statusCode }
386 const followersCount = await fetchActorTotalItems(actorJSON.followers)
387 const followingCount = await fetchActorTotalItems(actorJSON.following)
389 const actor = new ActorModel({
390 type: actorJSON.type,
391 uuid: actorJSON.uuid,
392 preferredUsername: actorJSON.preferredUsername,
394 publicKey: actorJSON.publicKey.publicKeyPem,
396 followersCount: followersCount,
397 followingCount: followingCount,
398 inboxUrl: actorJSON.inbox,
399 outboxUrl: actorJSON.outbox,
400 sharedInboxUrl: actorJSON.endpoints.sharedInbox,
401 followersUrl: actorJSON.followers,
402 followingUrl: actorJSON.following
405 const avatarName = await fetchAvatarIfExists(actorJSON)
407 const name = actorJSON.name || actorJSON.preferredUsername
409 statusCode: requestResult.response.statusCode,
414 summary: actorJSON.summary,
415 support: actorJSON.support,
416 playlists: actorJSON.playlists,
417 attributedTo: actorJSON.attributedTo
422 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
423 const [ accountCreated ] = await AccountModel.findOrCreate({
426 description: result.summary,
435 return accountCreated
438 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
439 const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
442 description: result.summary,
443 support: result.support,
445 accountId: ownerActor.Account.id
453 return videoChannelCreated