server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24
  25 // Set account keys, this could be long so process after the account creation and do not block the client
  26 function setAsyncActorKeys (actor: ActorModel) {
  27   return createPrivateAndPublicKeys()
  28     .then(({ publicKey, privateKey }) => {
  29       actor.set('publicKey', publicKey)
  30       actor.set('privateKey', privateKey)
  31       return actor.save()
  32     })
  33     .catch(err => {
  34       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  35       return actor
  36     })
  37 }
  38
  39 async function getOrCreateActorAndServerAndModel (
  40   activityActor: string | ActivityPubActor,
  41   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  42   recurseIfNeeded = true,
  43   updateCollections = false
  44 ) {
  45   const actorUrl = getAPUrl(activityActor)
  46   let created = false
  47
  48   let actor = await fetchActorByUrl(actorUrl, fetchType)
  49   // Orphan actor (not associated to an account of channel) so recreate it
  50   if (actor && (!actor.Account && !actor.VideoChannel)) {
  51     await actor.destroy()
  52     actor = null
  53   }
  54
  55   // We don't have this actor in our database, fetch it on remote
  56   if (!actor) {
  57     const { result } = await fetchRemoteActor(actorUrl)
  58     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  59
  60     // Create the attributed to actor
  61     // In PeerTube a video channel is owned by an account
  62     let ownerActor: ActorModel = undefined
  63     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  64       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  65       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  66
  67       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  68         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  69       }
  70
  71       try {
  72         // Don't recurse another time
  73         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
  74       } catch (err) {
  75         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  76         throw new Error(err)
  77       }
  78     }
  79
  80     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  81     created = true
  82   }
  83
  84   if (actor.Account) actor.Account.Actor = actor
  85   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  86
  87   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  88   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  89
  90   if ((created === true || refreshed === true) && updateCollections === true) {
  91     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  92     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  93   }
  94
  95   return actorRefreshed
  96 }
  97
  98 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
  99   return new ActorModel({
 100     type,
 101     url,
 102     preferredUsername,
 103     uuid,
 104     publicKey: null,
 105     privateKey: null,
 106     followersCount: 0,
 107     followingCount: 0,
 108     inboxUrl: url + '/inbox',
 109     outboxUrl: url + '/outbox',
 110     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 111     followersUrl: url + '/followers',
 112     followingUrl: url + '/following'
 113   })
 114 }
 115
 116 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 117   const followersCount = await fetchActorTotalItems(attributes.followers)
 118   const followingCount = await fetchActorTotalItems(attributes.following)
 119
 120   actorInstance.set('type', attributes.type)
 121   actorInstance.set('uuid', attributes.uuid)
 122   actorInstance.set('preferredUsername', attributes.preferredUsername)
 123   actorInstance.set('url', attributes.id)
 124   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 125   actorInstance.set('followersCount', followersCount)
 126   actorInstance.set('followingCount', followingCount)
 127   actorInstance.set('inboxUrl', attributes.inbox)
 128   actorInstance.set('outboxUrl', attributes.outbox)
 129   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 130   actorInstance.set('followersUrl', attributes.followers)
 131   actorInstance.set('followingUrl', attributes.following)
 132 }
 133
 134 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 135   if (avatarName !== undefined) {
 136     if (actorInstance.avatarId) {
 137       try {
 138         await actorInstance.Avatar.destroy({ transaction: t })
 139       } catch (err) {
 140         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 141       }
 142     }
 143
 144     const avatar = await AvatarModel.create({
 145       filename: avatarName
 146     }, { transaction: t })
 147
 148     actorInstance.set('avatarId', avatar.id)
 149     actorInstance.Avatar = avatar
 150   }
 151
 152   return actorInstance
 153 }
 154
 155 async function fetchActorTotalItems (url: string) {
 156   const options = {
 157     uri: url,
 158     method: 'GET',
 159     json: true,
 160     activityPub: true
 161   }
 162
 163   try {
 164     const { body } = await doRequest(options)
 165     return body.totalItems ? body.totalItems : 0
 166   } catch (err) {
 167     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 168     return 0
 169   }
 170 }
 171
 172 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 173   if (
 174     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 175     isActivityPubUrlValid(actorJSON.icon.url)
 176   ) {
 177     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 178
 179     const avatarName = uuidv4() + extension
 180     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 181
 182     return avatarName
 183   }
 184
 185   return undefined
 186 }
 187
 188 async function addFetchOutboxJob (actor: ActorModel) {
 189   // Don't fetch ourselves
 190   const serverActor = await getServerActor()
 191   if (serverActor.id === actor.id) {
 192     logger.error('Cannot fetch our own outbox!')
 193     return undefined
 194   }
 195
 196   const payload = {
 197     uri: actor.outboxUrl,
 198     type: 'activity' as 'activity'
 199   }
 200
 201   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 202 }
 203
 204 async function refreshActorIfNeeded (
 205   actorArg: ActorModel,
 206   fetchedType: ActorFetchByUrlType
 207 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 208   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 209
 210   // We need more attributes
 211   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 212
 213   try {
 214     const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 215     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 216
 217     if (statusCode === 404) {
 218       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 219       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 220       return { actor: undefined, refreshed: false }
 221     }
 222
 223     if (result === undefined) {
 224       logger.warn('Cannot fetch remote actor in refresh actor.')
 225       return { actor, refreshed: false }
 226     }
 227
 228     return sequelizeTypescript.transaction(async t => {
 229       updateInstanceWithAnother(actor, result.actor)
 230
 231       if (result.avatarName !== undefined) {
 232         await updateActorAvatarInstance(actor, result.avatarName, t)
 233       }
 234
 235       // Force update
 236       actor.setDataValue('updatedAt', new Date())
 237       await actor.save({ transaction: t })
 238
 239       if (actor.Account) {
 240         actor.Account.set('name', result.name)
 241         actor.Account.set('description', result.summary)
 242
 243         await actor.Account.save({ transaction: t })
 244       } else if (actor.VideoChannel) {
 245         actor.VideoChannel.set('name', result.name)
 246         actor.VideoChannel.set('description', result.summary)
 247         actor.VideoChannel.set('support', result.support)
 248
 249         await actor.VideoChannel.save({ transaction: t })
 250       }
 251
 252       return { refreshed: true, actor }
 253     })
 254   } catch (err) {
 255     logger.warn('Cannot refresh actor.', { err })
 256     return { actor, refreshed: false }
 257   }
 258 }
 259
 260 export {
 261   getOrCreateActorAndServerAndModel,
 262   buildActorInstance,
 263   setAsyncActorKeys,
 264   fetchActorTotalItems,
 265   fetchAvatarIfExists,
 266   updateActorInstance,
 267   refreshActorIfNeeded,
 268   updateActorAvatarInstance,
 269   addFetchOutboxJob
 270 }
 271
 272 // ---------------------------------------------------------------------------
 273
 274 function saveActorAndServerAndModelIfNotExist (
 275   result: FetchRemoteActorResult,
 276   ownerActor?: ActorModel,
 277   t?: Transaction
 278 ): Bluebird<ActorModel> | Promise<ActorModel> {
 279   let actor = result.actor
 280
 281   if (t !== undefined) return save(t)
 282
 283   return sequelizeTypescript.transaction(t => save(t))
 284
 285   async function save (t: Transaction) {
 286     const actorHost = url.parse(actor.url).host
 287
 288     const serverOptions = {
 289       where: {
 290         host: actorHost
 291       },
 292       defaults: {
 293         host: actorHost
 294       },
 295       transaction: t
 296     }
 297     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 298
 299     // Save our new account in database
 300     actor.set('serverId', server.id)
 301
 302     // Avatar?
 303     if (result.avatarName) {
 304       const avatar = await AvatarModel.create({
 305         filename: result.avatarName
 306       }, { transaction: t })
 307       actor.set('avatarId', avatar.id)
 308     }
 309
 310     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 311     // (which could be false in a retried query)
 312     const [ actorCreated ] = await ActorModel.findOrCreate({
 313       defaults: actor.toJSON(),
 314       where: {
 315         url: actor.url
 316       },
 317       transaction: t
 318     })
 319
 320     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 321       actorCreated.Account = await saveAccount(actorCreated, result, t)
 322       actorCreated.Account.Actor = actorCreated
 323     } else if (actorCreated.type === 'Group') { // Video channel
 324       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 325       actorCreated.VideoChannel.Actor = actorCreated
 326       actorCreated.VideoChannel.Account = ownerActor.Account
 327     }
 328
 329     return actorCreated
 330   }
 331 }
 332
 333 type FetchRemoteActorResult = {
 334   actor: ActorModel
 335   name: string
 336   summary: string
 337   support?: string
 338   avatarName?: string
 339   attributedTo: ActivityPubAttributedTo[]
 340 }
 341 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 342   const options = {
 343     uri: actorUrl,
 344     method: 'GET',
 345     json: true,
 346     activityPub: true
 347   }
 348
 349   logger.info('Fetching remote actor %s.', actorUrl)
 350
 351   const requestResult = await doRequest(options)
 352   normalizeActor(requestResult.body)
 353
 354   const actorJSON: ActivityPubActor = requestResult.body
 355   if (isActorObjectValid(actorJSON) === false) {
 356     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 357     return { result: undefined, statusCode: requestResult.response.statusCode }
 358   }
 359
 360   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 361     throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
 362   }
 363
 364   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 365   const followingCount = await fetchActorTotalItems(actorJSON.following)
 366
 367   const actor = new ActorModel({
 368     type: actorJSON.type,
 369     uuid: actorJSON.uuid,
 370     preferredUsername: actorJSON.preferredUsername,
 371     url: actorJSON.id,
 372     publicKey: actorJSON.publicKey.publicKeyPem,
 373     privateKey: null,
 374     followersCount: followersCount,
 375     followingCount: followingCount,
 376     inboxUrl: actorJSON.inbox,
 377     outboxUrl: actorJSON.outbox,
 378     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 379     followersUrl: actorJSON.followers,
 380     followingUrl: actorJSON.following
 381   })
 382
 383   const avatarName = await fetchAvatarIfExists(actorJSON)
 384
 385   const name = actorJSON.name || actorJSON.preferredUsername
 386   return {
 387     statusCode: requestResult.response.statusCode,
 388     result: {
 389       actor,
 390       name,
 391       avatarName,
 392       summary: actorJSON.summary,
 393       support: actorJSON.support,
 394       attributedTo: actorJSON.attributedTo
 395     }
 396   }
 397 }
 398
 399 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 400   const [ accountCreated ] = await AccountModel.findOrCreate({
 401     defaults: {
 402       name: result.name,
 403       description: result.summary,
 404       actorId: actor.id
 405     },
 406     where: {
 407       actorId: actor.id
 408     },
 409     transaction: t
 410   })
 411
 412   return accountCreated
 413 }
 414
 415 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 416   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 417     defaults: {
 418       name: result.name,
 419       description: result.summary,
 420       support: result.support,
 421       actorId: actor.id,
 422       accountId: ownerActor.Account.id
 423     },
 424     where: {
 425       actorId: actor.id
 426     },
 427     transaction: t
 428   })
 429
 430   return videoChannelCreated
 431 }
 432
 433