server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { join } from 'path'
   3 import { Transaction } from 'sequelize'
   4 import * as url from 'url'
   5 import * as uuidv4 from 'uuid/v4'
   6 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   7 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   8 import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
   9 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
  10 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  11 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  12 import { logger } from '../../helpers/logger'
  13 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  14 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
  15 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  16 import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers'
  17 import { AccountModel } from '../../models/account/account'
  18 import { ActorModel } from '../../models/activitypub/actor'
  19 import { AvatarModel } from '../../models/avatar/avatar'
  20 import { ServerModel } from '../../models/server/server'
  21 import { VideoChannelModel } from '../../models/video/video-channel'
  22 import { JobQueue } from '../job-queue'
  23 import { getServerActor } from '../../helpers/utils'
  24 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  25
  26 // Set account keys, this could be long so process after the account creation and do not block the client
  27 function setAsyncActorKeys (actor: ActorModel) {
  28   return createPrivateAndPublicKeys()
  29     .then(({ publicKey, privateKey }) => {
  30       actor.set('publicKey', publicKey)
  31       actor.set('privateKey', privateKey)
  32       return actor.save()
  33     })
  34     .catch(err => {
  35       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  36       return actor
  37     })
  38 }
  39
  40 async function getOrCreateActorAndServerAndModel (
  41   activityActor: string | ActivityPubActor,
  42   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  43   recurseIfNeeded = true,
  44   updateCollections = false
  45 ) {
  46   const actorUrl = getActorUrl(activityActor)
  47   let created = false
  48
  49   let actor = await fetchActorByUrl(actorUrl, fetchType)
  50   // Orphan actor (not associated to an account of channel) so recreate it
  51   if (actor && (!actor.Account && !actor.VideoChannel)) {
  52     await actor.destroy()
  53     actor = null
  54   }
  55
  56   // We don't have this actor in our database, fetch it on remote
  57   if (!actor) {
  58     const { result } = await fetchRemoteActor(actorUrl)
  59     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  60
  61     // Create the attributed to actor
  62     // In PeerTube a video channel is owned by an account
  63     let ownerActor: ActorModel = undefined
  64     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  65       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  66       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  67
  68       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  69         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  70       }
  71
  72       try {
  73         // Don't recurse another time
  74         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
  75       } catch (err) {
  76         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  77         throw new Error(err)
  78       }
  79     }
  80
  81     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  82     created = true
  83   }
  84
  85   if (actor.Account) actor.Account.Actor = actor
  86   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  87
  88   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  89   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  90
  91   if ((created === true || refreshed === true) && updateCollections === true) {
  92     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  93     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  94   }
  95
  96   return actorRefreshed
  97 }
  98
  99 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 100   return new ActorModel({
 101     type,
 102     url,
 103     preferredUsername,
 104     uuid,
 105     publicKey: null,
 106     privateKey: null,
 107     followersCount: 0,
 108     followingCount: 0,
 109     inboxUrl: url + '/inbox',
 110     outboxUrl: url + '/outbox',
 111     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 112     followersUrl: url + '/followers',
 113     followingUrl: url + '/following'
 114   })
 115 }
 116
 117 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 118   const followersCount = await fetchActorTotalItems(attributes.followers)
 119   const followingCount = await fetchActorTotalItems(attributes.following)
 120
 121   actorInstance.set('type', attributes.type)
 122   actorInstance.set('uuid', attributes.uuid)
 123   actorInstance.set('preferredUsername', attributes.preferredUsername)
 124   actorInstance.set('url', attributes.id)
 125   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 126   actorInstance.set('followersCount', followersCount)
 127   actorInstance.set('followingCount', followingCount)
 128   actorInstance.set('inboxUrl', attributes.inbox)
 129   actorInstance.set('outboxUrl', attributes.outbox)
 130   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 131   actorInstance.set('followersUrl', attributes.followers)
 132   actorInstance.set('followingUrl', attributes.following)
 133 }
 134
 135 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 136   if (avatarName !== undefined) {
 137     if (actorInstance.avatarId) {
 138       try {
 139         await actorInstance.Avatar.destroy({ transaction: t })
 140       } catch (err) {
 141         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 142       }
 143     }
 144
 145     const avatar = await AvatarModel.create({
 146       filename: avatarName
 147     }, { transaction: t })
 148
 149     actorInstance.set('avatarId', avatar.id)
 150     actorInstance.Avatar = avatar
 151   }
 152
 153   return actorInstance
 154 }
 155
 156 async function fetchActorTotalItems (url: string) {
 157   const options = {
 158     uri: url,
 159     method: 'GET',
 160     json: true,
 161     activityPub: true
 162   }
 163
 164   try {
 165     const { body } = await doRequest(options)
 166     return body.totalItems ? body.totalItems : 0
 167   } catch (err) {
 168     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 169     return 0
 170   }
 171 }
 172
 173 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 174   if (
 175     actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 176     isActivityPubUrlValid(actorJSON.icon.url)
 177   ) {
 178     const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
 179
 180     const avatarName = uuidv4() + extension
 181     const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
 182
 183     await doRequestAndSaveToFile({
 184       method: 'GET',
 185       uri: actorJSON.icon.url
 186     }, destPath)
 187
 188     return avatarName
 189   }
 190
 191   return undefined
 192 }
 193
 194 async function addFetchOutboxJob (actor: ActorModel) {
 195   // Don't fetch ourselves
 196   const serverActor = await getServerActor()
 197   if (serverActor.id === actor.id) {
 198     logger.error('Cannot fetch our own outbox!')
 199     return undefined
 200   }
 201
 202   const payload = {
 203     uri: actor.outboxUrl,
 204     type: 'activity' as 'activity'
 205   }
 206
 207   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 208 }
 209
 210 export {
 211   getOrCreateActorAndServerAndModel,
 212   buildActorInstance,
 213   setAsyncActorKeys,
 214   fetchActorTotalItems,
 215   fetchAvatarIfExists,
 216   updateActorInstance,
 217   updateActorAvatarInstance,
 218   addFetchOutboxJob
 219 }
 220
 221 // ---------------------------------------------------------------------------
 222
 223 function saveActorAndServerAndModelIfNotExist (
 224   result: FetchRemoteActorResult,
 225   ownerActor?: ActorModel,
 226   t?: Transaction
 227 ): Bluebird<ActorModel> | Promise<ActorModel> {
 228   let actor = result.actor
 229
 230   if (t !== undefined) return save(t)
 231
 232   return sequelizeTypescript.transaction(t => save(t))
 233
 234   async function save (t: Transaction) {
 235     const actorHost = url.parse(actor.url).host
 236
 237     const serverOptions = {
 238       where: {
 239         host: actorHost
 240       },
 241       defaults: {
 242         host: actorHost
 243       },
 244       transaction: t
 245     }
 246     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 247
 248     // Save our new account in database
 249     actor.set('serverId', server.id)
 250
 251     // Avatar?
 252     if (result.avatarName) {
 253       const avatar = await AvatarModel.create({
 254         filename: result.avatarName
 255       }, { transaction: t })
 256       actor.set('avatarId', avatar.id)
 257     }
 258
 259     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 260     // (which could be false in a retried query)
 261     const [ actorCreated ] = await ActorModel.findOrCreate({
 262       defaults: actor.toJSON(),
 263       where: {
 264         url: actor.url
 265       },
 266       transaction: t
 267     })
 268
 269     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 270       actorCreated.Account = await saveAccount(actorCreated, result, t)
 271       actorCreated.Account.Actor = actorCreated
 272     } else if (actorCreated.type === 'Group') { // Video channel
 273       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 274       actorCreated.VideoChannel.Actor = actorCreated
 275       actorCreated.VideoChannel.Account = ownerActor.Account
 276     }
 277
 278     return actorCreated
 279   }
 280 }
 281
 282 type FetchRemoteActorResult = {
 283   actor: ActorModel
 284   name: string
 285   summary: string
 286   support?: string
 287   avatarName?: string
 288   attributedTo: ActivityPubAttributedTo[]
 289 }
 290 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 291   const options = {
 292     uri: actorUrl,
 293     method: 'GET',
 294     json: true,
 295     activityPub: true
 296   }
 297
 298   logger.info('Fetching remote actor %s.', actorUrl)
 299
 300   const requestResult = await doRequest(options)
 301   normalizeActor(requestResult.body)
 302
 303   const actorJSON: ActivityPubActor = requestResult.body
 304   if (isActorObjectValid(actorJSON) === false) {
 305     logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
 306     return { result: undefined, statusCode: requestResult.response.statusCode }
 307   }
 308
 309   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 310     throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
 311   }
 312
 313   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 314   const followingCount = await fetchActorTotalItems(actorJSON.following)
 315
 316   const actor = new ActorModel({
 317     type: actorJSON.type,
 318     uuid: actorJSON.uuid,
 319     preferredUsername: actorJSON.preferredUsername,
 320     url: actorJSON.id,
 321     publicKey: actorJSON.publicKey.publicKeyPem,
 322     privateKey: null,
 323     followersCount: followersCount,
 324     followingCount: followingCount,
 325     inboxUrl: actorJSON.inbox,
 326     outboxUrl: actorJSON.outbox,
 327     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 328     followersUrl: actorJSON.followers,
 329     followingUrl: actorJSON.following
 330   })
 331
 332   const avatarName = await fetchAvatarIfExists(actorJSON)
 333
 334   const name = actorJSON.name || actorJSON.preferredUsername
 335   return {
 336     statusCode: requestResult.response.statusCode,
 337     result: {
 338       actor,
 339       name,
 340       avatarName,
 341       summary: actorJSON.summary,
 342       support: actorJSON.support,
 343       attributedTo: actorJSON.attributedTo
 344     }
 345   }
 346 }
 347
 348 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 349   const [ accountCreated ] = await AccountModel.findOrCreate({
 350     defaults: {
 351       name: result.name,
 352       description: result.summary,
 353       actorId: actor.id
 354     },
 355     where: {
 356       actorId: actor.id
 357     },
 358     transaction: t
 359   })
 360
 361   return accountCreated
 362 }
 363
 364 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 365   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 366     defaults: {
 367       name: result.name,
 368       description: result.summary,
 369       support: result.support,
 370       actorId: actor.id,
 371       accountId: ownerActor.Account.id
 372     },
 373     where: {
 374       actorId: actor.id
 375     },
 376     transaction: t
 377   })
 378
 379   return videoChannelCreated
 380 }
 381
 382 async function refreshActorIfNeeded (
 383   actorArg: ActorModel,
 384   fetchedType: ActorFetchByUrlType
 385 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 386   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 387
 388   // We need more attributes
 389   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 390
 391   try {
 392     const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 393     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 394
 395     if (statusCode === 404) {
 396       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 397       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 398       return { actor: undefined, refreshed: false }
 399     }
 400
 401     if (result === undefined) {
 402       logger.warn('Cannot fetch remote actor in refresh actor.')
 403       return { actor, refreshed: false }
 404     }
 405
 406     return sequelizeTypescript.transaction(async t => {
 407       updateInstanceWithAnother(actor, result.actor)
 408
 409       if (result.avatarName !== undefined) {
 410         await updateActorAvatarInstance(actor, result.avatarName, t)
 411       }
 412
 413       // Force update
 414       actor.setDataValue('updatedAt', new Date())
 415       await actor.save({ transaction: t })
 416
 417       if (actor.Account) {
 418         actor.Account.set('name', result.name)
 419         actor.Account.set('description', result.summary)
 420
 421         await actor.Account.save({ transaction: t })
 422       } else if (actor.VideoChannel) {
 423         actor.VideoChannel.set('name', result.name)
 424         actor.VideoChannel.set('description', result.summary)
 425         actor.VideoChannel.set('support', result.support)
 426
 427         await actor.VideoChannel.save({ transaction: t })
 428       }
 429
 430       return { refreshed: true, actor }
 431     })
 432   } catch (err) {
 433     logger.warn('Cannot refresh actor.', { err })
 434     return { actor, refreshed: false }
 435   }
 436 }