server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24
  25 // Set account keys, this could be long so process after the account creation and do not block the client
  26 function setAsyncActorKeys (actor: ActorModel) {
  27   return createPrivateAndPublicKeys()
  28     .then(({ publicKey, privateKey }) => {
  29       actor.set('publicKey', publicKey)
  30       actor.set('privateKey', privateKey)
  31       return actor.save()
  32     })
  33     .catch(err => {
  34       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  35       return actor
  36     })
  37 }
  38
  39 async function getOrCreateActorAndServerAndModel (
  40   activityActor: string | ActivityPubActor,
  41   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  42   recurseIfNeeded = true,
  43   updateCollections = false
  44 ) {
  45   const actorUrl = getAPUrl(activityActor)
  46   let created = false
  47
  48   let actor = await fetchActorByUrl(actorUrl, fetchType)
  49   // Orphan actor (not associated to an account of channel) so recreate it
  50   if (actor && (!actor.Account && !actor.VideoChannel)) {
  51     await actor.destroy()
  52     actor = null
  53   }
  54
  55   // We don't have this actor in our database, fetch it on remote
  56   if (!actor) {
  57     const { result } = await fetchRemoteActor(actorUrl)
  58     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  59
  60     // Create the attributed to actor
  61     // In PeerTube a video channel is owned by an account
  62     let ownerActor: ActorModel = undefined
  63     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  64       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  65       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  66
  67       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  68         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  69       }
  70
  71       try {
  72         // Don't recurse another time
  73         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
  74       } catch (err) {
  75         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  76         throw new Error(err)
  77       }
  78     }
  79
  80     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  81     created = true
  82   }
  83
  84   if (actor.Account) actor.Account.Actor = actor
  85   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  86
  87   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  88   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  89
  90   if ((created === true || refreshed === true) && updateCollections === true) {
  91     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  92     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  93   }
  94
  95   return actorRefreshed
  96 }
  97
  98 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
  99   return new ActorModel({
 100     type,
 101     url,
 102     preferredUsername,
 103     uuid,
 104     publicKey: null,
 105     privateKey: null,
 106     followersCount: 0,
 107     followingCount: 0,
 108     inboxUrl: url + '/inbox',
 109     outboxUrl: url + '/outbox',
 110     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 111     followersUrl: url + '/followers',
 112     followingUrl: url + '/following'
 113   })
 114 }
 115
 116 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 117   const followersCount = await fetchActorTotalItems(attributes.followers)
 118   const followingCount = await fetchActorTotalItems(attributes.following)
 119
 120   actorInstance.set('type', attributes.type)
 121   actorInstance.set('uuid', attributes.uuid)
 122   actorInstance.set('preferredUsername', attributes.preferredUsername)
 123   actorInstance.set('url', attributes.id)
 124   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 125   actorInstance.set('followersCount', followersCount)
 126   actorInstance.set('followingCount', followingCount)
 127   actorInstance.set('inboxUrl', attributes.inbox)
 128   actorInstance.set('outboxUrl', attributes.outbox)
 129   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 130   actorInstance.set('followersUrl', attributes.followers)
 131   actorInstance.set('followingUrl', attributes.following)
 132 }
 133
 134 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 135   if (avatarName !== undefined) {
 136     if (actorInstance.avatarId) {
 137       try {
 138         await actorInstance.Avatar.destroy({ transaction: t })
 139       } catch (err) {
 140         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 141       }
 142     }
 143
 144     const avatar = await AvatarModel.create({
 145       filename: avatarName
 146     }, { transaction: t })
 147
 148     actorInstance.set('avatarId', avatar.id)
 149     actorInstance.Avatar = avatar
 150   }
 151
 152   return actorInstance
 153 }
 154
 155 async function fetchActorTotalItems (url: string) {
 156   const options = {
 157     uri: url,
 158     method: 'GET',
 159     json: true,
 160     activityPub: true
 161   }
 162
 163   try {
 164     const { body } = await doRequest(options)
 165     return body.totalItems ? body.totalItems : 0
 166   } catch (err) {
 167     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 168     return 0
 169   }
 170 }
 171
 172 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 173   if (
 174     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 175     isActivityPubUrlValid(actorJSON.icon.url)
 176   ) {
 177     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 178
 179     const avatarName = uuidv4() + extension
 180     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 181
 182     return avatarName
 183   }
 184
 185   return undefined
 186 }
 187
 188 async function addFetchOutboxJob (actor: ActorModel) {
 189   // Don't fetch ourselves
 190   const serverActor = await getServerActor()
 191   if (serverActor.id === actor.id) {
 192     logger.error('Cannot fetch our own outbox!')
 193     return undefined
 194   }
 195
 196   const payload = {
 197     uri: actor.outboxUrl,
 198     type: 'activity' as 'activity'
 199   }
 200
 201   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 202 }
 203
 204 export {
 205   getOrCreateActorAndServerAndModel,
 206   buildActorInstance,
 207   setAsyncActorKeys,
 208   fetchActorTotalItems,
 209   fetchAvatarIfExists,
 210   updateActorInstance,
 211   updateActorAvatarInstance,
 212   addFetchOutboxJob
 213 }
 214
 215 // ---------------------------------------------------------------------------
 216
 217 function saveActorAndServerAndModelIfNotExist (
 218   result: FetchRemoteActorResult,
 219   ownerActor?: ActorModel,
 220   t?: Transaction
 221 ): Bluebird<ActorModel> | Promise<ActorModel> {
 222   let actor = result.actor
 223
 224   if (t !== undefined) return save(t)
 225
 226   return sequelizeTypescript.transaction(t => save(t))
 227
 228   async function save (t: Transaction) {
 229     const actorHost = url.parse(actor.url).host
 230
 231     const serverOptions = {
 232       where: {
 233         host: actorHost
 234       },
 235       defaults: {
 236         host: actorHost
 237       },
 238       transaction: t
 239     }
 240     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 241
 242     // Save our new account in database
 243     actor.set('serverId', server.id)
 244
 245     // Avatar?
 246     if (result.avatarName) {
 247       const avatar = await AvatarModel.create({
 248         filename: result.avatarName
 249       }, { transaction: t })
 250       actor.set('avatarId', avatar.id)
 251     }
 252
 253     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 254     // (which could be false in a retried query)
 255     const [ actorCreated ] = await ActorModel.findOrCreate({
 256       defaults: actor.toJSON(),
 257       where: {
 258         url: actor.url
 259       },
 260       transaction: t
 261     })
 262
 263     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 264       actorCreated.Account = await saveAccount(actorCreated, result, t)
 265       actorCreated.Account.Actor = actorCreated
 266     } else if (actorCreated.type === 'Group') { // Video channel
 267       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 268       actorCreated.VideoChannel.Actor = actorCreated
 269       actorCreated.VideoChannel.Account = ownerActor.Account
 270     }
 271
 272     return actorCreated
 273   }
 274 }
 275
 276 type FetchRemoteActorResult = {
 277   actor: ActorModel
 278   name: string
 279   summary: string
 280   support?: string
 281   avatarName?: string
 282   attributedTo: ActivityPubAttributedTo[]
 283 }
 284 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 285   const options = {
 286     uri: actorUrl,
 287     method: 'GET',
 288     json: true,
 289     activityPub: true
 290   }
 291
 292   logger.info('Fetching remote actor %s.', actorUrl)
 293
 294   const requestResult = await doRequest(options)
 295   normalizeActor(requestResult.body)
 296
 297   const actorJSON: ActivityPubActor = requestResult.body
 298   if (isActorObjectValid(actorJSON) === false) {
 299     logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
 300     return { result: undefined, statusCode: requestResult.response.statusCode }
 301   }
 302
 303   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 304     throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
 305   }
 306
 307   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 308   const followingCount = await fetchActorTotalItems(actorJSON.following)
 309
 310   const actor = new ActorModel({
 311     type: actorJSON.type,
 312     uuid: actorJSON.uuid,
 313     preferredUsername: actorJSON.preferredUsername,
 314     url: actorJSON.id,
 315     publicKey: actorJSON.publicKey.publicKeyPem,
 316     privateKey: null,
 317     followersCount: followersCount,
 318     followingCount: followingCount,
 319     inboxUrl: actorJSON.inbox,
 320     outboxUrl: actorJSON.outbox,
 321     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 322     followersUrl: actorJSON.followers,
 323     followingUrl: actorJSON.following
 324   })
 325
 326   const avatarName = await fetchAvatarIfExists(actorJSON)
 327
 328   const name = actorJSON.name || actorJSON.preferredUsername
 329   return {
 330     statusCode: requestResult.response.statusCode,
 331     result: {
 332       actor,
 333       name,
 334       avatarName,
 335       summary: actorJSON.summary,
 336       support: actorJSON.support,
 337       attributedTo: actorJSON.attributedTo
 338     }
 339   }
 340 }
 341
 342 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 343   const [ accountCreated ] = await AccountModel.findOrCreate({
 344     defaults: {
 345       name: result.name,
 346       description: result.summary,
 347       actorId: actor.id
 348     },
 349     where: {
 350       actorId: actor.id
 351     },
 352     transaction: t
 353   })
 354
 355   return accountCreated
 356 }
 357
 358 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 359   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 360     defaults: {
 361       name: result.name,
 362       description: result.summary,
 363       support: result.support,
 364       actorId: actor.id,
 365       accountId: ownerActor.Account.id
 366     },
 367     where: {
 368       actorId: actor.id
 369     },
 370     transaction: t
 371   })
 372
 373   return videoChannelCreated
 374 }
 375
 376 async function refreshActorIfNeeded (
 377   actorArg: ActorModel,
 378   fetchedType: ActorFetchByUrlType
 379 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 380   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 381
 382   // We need more attributes
 383   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 384
 385   try {
 386     const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 387     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 388
 389     if (statusCode === 404) {
 390       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 391       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 392       return { actor: undefined, refreshed: false }
 393     }
 394
 395     if (result === undefined) {
 396       logger.warn('Cannot fetch remote actor in refresh actor.')
 397       return { actor, refreshed: false }
 398     }
 399
 400     return sequelizeTypescript.transaction(async t => {
 401       updateInstanceWithAnother(actor, result.actor)
 402
 403       if (result.avatarName !== undefined) {
 404         await updateActorAvatarInstance(actor, result.avatarName, t)
 405       }
 406
 407       // Force update
 408       actor.setDataValue('updatedAt', new Date())
 409       await actor.save({ transaction: t })
 410
 411       if (actor.Account) {
 412         actor.Account.set('name', result.name)
 413         actor.Account.set('description', result.summary)
 414
 415         await actor.Account.save({ transaction: t })
 416       } else if (actor.VideoChannel) {
 417         actor.VideoChannel.set('name', result.name)
 418         actor.VideoChannel.set('description', result.summary)
 419         actor.VideoChannel.set('support', result.support)
 420
 421         await actor.VideoChannel.save({ transaction: t })
 422       }
 423
 424       return { refreshed: true, actor }
 425     })
 426   } catch (err) {
 427     logger.warn('Cannot refresh actor.', { err })
 428     return { actor, refreshed: false }
 429   }
 430 }