server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { join } from 'path'
   3 import { Transaction } from 'sequelize'
   4 import * as url from 'url'
   5 import * as uuidv4 from 'uuid/v4'
   6 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   7 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   8 import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
   9 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
  10 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  11 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  12 import { logger } from '../../helpers/logger'
  13 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  14 import { doRequest, doRequestAndSaveToFile, downloadImage } from '../../helpers/requests'
  15 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  16 import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript } from '../../initializers'
  17 import { AccountModel } from '../../models/account/account'
  18 import { ActorModel } from '../../models/activitypub/actor'
  19 import { AvatarModel } from '../../models/avatar/avatar'
  20 import { ServerModel } from '../../models/server/server'
  21 import { VideoChannelModel } from '../../models/video/video-channel'
  22 import { JobQueue } from '../job-queue'
  23 import { getServerActor } from '../../helpers/utils'
  24 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  25
  26 // Set account keys, this could be long so process after the account creation and do not block the client
  27 function setAsyncActorKeys (actor: ActorModel) {
  28   return createPrivateAndPublicKeys()
  29     .then(({ publicKey, privateKey }) => {
  30       actor.set('publicKey', publicKey)
  31       actor.set('privateKey', privateKey)
  32       return actor.save()
  33     })
  34     .catch(err => {
  35       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  36       return actor
  37     })
  38 }
  39
  40 async function getOrCreateActorAndServerAndModel (
  41   activityActor: string | ActivityPubActor,
  42   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  43   recurseIfNeeded = true,
  44   updateCollections = false
  45 ) {
  46   const actorUrl = getActorUrl(activityActor)
  47   let created = false
  48
  49   let actor = await fetchActorByUrl(actorUrl, fetchType)
  50   // Orphan actor (not associated to an account of channel) so recreate it
  51   if (actor && (!actor.Account && !actor.VideoChannel)) {
  52     await actor.destroy()
  53     actor = null
  54   }
  55
  56   // We don't have this actor in our database, fetch it on remote
  57   if (!actor) {
  58     const { result } = await fetchRemoteActor(actorUrl)
  59     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  60
  61     // Create the attributed to actor
  62     // In PeerTube a video channel is owned by an account
  63     let ownerActor: ActorModel = undefined
  64     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  65       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  66       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  67
  68       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  69         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  70       }
  71
  72       try {
  73         // Don't recurse another time
  74         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
  75       } catch (err) {
  76         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  77         throw new Error(err)
  78       }
  79     }
  80
  81     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  82     created = true
  83   }
  84
  85   if (actor.Account) actor.Account.Actor = actor
  86   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  87
  88   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  89   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  90
  91   if ((created === true || refreshed === true) && updateCollections === true) {
  92     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  93     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  94   }
  95
  96   return actorRefreshed
  97 }
  98
  99 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 100   return new ActorModel({
 101     type,
 102     url,
 103     preferredUsername,
 104     uuid,
 105     publicKey: null,
 106     privateKey: null,
 107     followersCount: 0,
 108     followingCount: 0,
 109     inboxUrl: url + '/inbox',
 110     outboxUrl: url + '/outbox',
 111     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 112     followersUrl: url + '/followers',
 113     followingUrl: url + '/following'
 114   })
 115 }
 116
 117 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 118   const followersCount = await fetchActorTotalItems(attributes.followers)
 119   const followingCount = await fetchActorTotalItems(attributes.following)
 120
 121   actorInstance.set('type', attributes.type)
 122   actorInstance.set('uuid', attributes.uuid)
 123   actorInstance.set('preferredUsername', attributes.preferredUsername)
 124   actorInstance.set('url', attributes.id)
 125   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 126   actorInstance.set('followersCount', followersCount)
 127   actorInstance.set('followingCount', followingCount)
 128   actorInstance.set('inboxUrl', attributes.inbox)
 129   actorInstance.set('outboxUrl', attributes.outbox)
 130   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 131   actorInstance.set('followersUrl', attributes.followers)
 132   actorInstance.set('followingUrl', attributes.following)
 133 }
 134
 135 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 136   if (avatarName !== undefined) {
 137     if (actorInstance.avatarId) {
 138       try {
 139         await actorInstance.Avatar.destroy({ transaction: t })
 140       } catch (err) {
 141         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 142       }
 143     }
 144
 145     const avatar = await AvatarModel.create({
 146       filename: avatarName
 147     }, { transaction: t })
 148
 149     actorInstance.set('avatarId', avatar.id)
 150     actorInstance.Avatar = avatar
 151   }
 152
 153   return actorInstance
 154 }
 155
 156 async function fetchActorTotalItems (url: string) {
 157   const options = {
 158     uri: url,
 159     method: 'GET',
 160     json: true,
 161     activityPub: true
 162   }
 163
 164   try {
 165     const { body } = await doRequest(options)
 166     return body.totalItems ? body.totalItems : 0
 167   } catch (err) {
 168     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 169     return 0
 170   }
 171 }
 172
 173 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 174   if (
 175     actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 176     isActivityPubUrlValid(actorJSON.icon.url)
 177   ) {
 178     const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
 179
 180     const avatarName = uuidv4() + extension
 181     const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
 182
 183     await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE)
 184
 185     return avatarName
 186   }
 187
 188   return undefined
 189 }
 190
 191 async function addFetchOutboxJob (actor: ActorModel) {
 192   // Don't fetch ourselves
 193   const serverActor = await getServerActor()
 194   if (serverActor.id === actor.id) {
 195     logger.error('Cannot fetch our own outbox!')
 196     return undefined
 197   }
 198
 199   const payload = {
 200     uri: actor.outboxUrl,
 201     type: 'activity' as 'activity'
 202   }
 203
 204   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 205 }
 206
 207 export {
 208   getOrCreateActorAndServerAndModel,
 209   buildActorInstance,
 210   setAsyncActorKeys,
 211   fetchActorTotalItems,
 212   fetchAvatarIfExists,
 213   updateActorInstance,
 214   updateActorAvatarInstance,
 215   addFetchOutboxJob
 216 }
 217
 218 // ---------------------------------------------------------------------------
 219
 220 function saveActorAndServerAndModelIfNotExist (
 221   result: FetchRemoteActorResult,
 222   ownerActor?: ActorModel,
 223   t?: Transaction
 224 ): Bluebird<ActorModel> | Promise<ActorModel> {
 225   let actor = result.actor
 226
 227   if (t !== undefined) return save(t)
 228
 229   return sequelizeTypescript.transaction(t => save(t))
 230
 231   async function save (t: Transaction) {
 232     const actorHost = url.parse(actor.url).host
 233
 234     const serverOptions = {
 235       where: {
 236         host: actorHost
 237       },
 238       defaults: {
 239         host: actorHost
 240       },
 241       transaction: t
 242     }
 243     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 244
 245     // Save our new account in database
 246     actor.set('serverId', server.id)
 247
 248     // Avatar?
 249     if (result.avatarName) {
 250       const avatar = await AvatarModel.create({
 251         filename: result.avatarName
 252       }, { transaction: t })
 253       actor.set('avatarId', avatar.id)
 254     }
 255
 256     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 257     // (which could be false in a retried query)
 258     const [ actorCreated ] = await ActorModel.findOrCreate({
 259       defaults: actor.toJSON(),
 260       where: {
 261         url: actor.url
 262       },
 263       transaction: t
 264     })
 265
 266     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 267       actorCreated.Account = await saveAccount(actorCreated, result, t)
 268       actorCreated.Account.Actor = actorCreated
 269     } else if (actorCreated.type === 'Group') { // Video channel
 270       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 271       actorCreated.VideoChannel.Actor = actorCreated
 272       actorCreated.VideoChannel.Account = ownerActor.Account
 273     }
 274
 275     return actorCreated
 276   }
 277 }
 278
 279 type FetchRemoteActorResult = {
 280   actor: ActorModel
 281   name: string
 282   summary: string
 283   support?: string
 284   avatarName?: string
 285   attributedTo: ActivityPubAttributedTo[]
 286 }
 287 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 288   const options = {
 289     uri: actorUrl,
 290     method: 'GET',
 291     json: true,
 292     activityPub: true
 293   }
 294
 295   logger.info('Fetching remote actor %s.', actorUrl)
 296
 297   const requestResult = await doRequest(options)
 298   normalizeActor(requestResult.body)
 299
 300   const actorJSON: ActivityPubActor = requestResult.body
 301   if (isActorObjectValid(actorJSON) === false) {
 302     logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
 303     return { result: undefined, statusCode: requestResult.response.statusCode }
 304   }
 305
 306   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 307     throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
 308   }
 309
 310   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 311   const followingCount = await fetchActorTotalItems(actorJSON.following)
 312
 313   const actor = new ActorModel({
 314     type: actorJSON.type,
 315     uuid: actorJSON.uuid,
 316     preferredUsername: actorJSON.preferredUsername,
 317     url: actorJSON.id,
 318     publicKey: actorJSON.publicKey.publicKeyPem,
 319     privateKey: null,
 320     followersCount: followersCount,
 321     followingCount: followingCount,
 322     inboxUrl: actorJSON.inbox,
 323     outboxUrl: actorJSON.outbox,
 324     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 325     followersUrl: actorJSON.followers,
 326     followingUrl: actorJSON.following
 327   })
 328
 329   const avatarName = await fetchAvatarIfExists(actorJSON)
 330
 331   const name = actorJSON.name || actorJSON.preferredUsername
 332   return {
 333     statusCode: requestResult.response.statusCode,
 334     result: {
 335       actor,
 336       name,
 337       avatarName,
 338       summary: actorJSON.summary,
 339       support: actorJSON.support,
 340       attributedTo: actorJSON.attributedTo
 341     }
 342   }
 343 }
 344
 345 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 346   const [ accountCreated ] = await AccountModel.findOrCreate({
 347     defaults: {
 348       name: result.name,
 349       description: result.summary,
 350       actorId: actor.id
 351     },
 352     where: {
 353       actorId: actor.id
 354     },
 355     transaction: t
 356   })
 357
 358   return accountCreated
 359 }
 360
 361 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 362   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 363     defaults: {
 364       name: result.name,
 365       description: result.summary,
 366       support: result.support,
 367       actorId: actor.id,
 368       accountId: ownerActor.Account.id
 369     },
 370     where: {
 371       actorId: actor.id
 372     },
 373     transaction: t
 374   })
 375
 376   return videoChannelCreated
 377 }
 378
 379 async function refreshActorIfNeeded (
 380   actorArg: ActorModel,
 381   fetchedType: ActorFetchByUrlType
 382 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 383   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 384
 385   // We need more attributes
 386   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 387
 388   try {
 389     const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 390     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 391
 392     if (statusCode === 404) {
 393       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 394       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 395       return { actor: undefined, refreshed: false }
 396     }
 397
 398     if (result === undefined) {
 399       logger.warn('Cannot fetch remote actor in refresh actor.')
 400       return { actor, refreshed: false }
 401     }
 402
 403     return sequelizeTypescript.transaction(async t => {
 404       updateInstanceWithAnother(actor, result.actor)
 405
 406       if (result.avatarName !== undefined) {
 407         await updateActorAvatarInstance(actor, result.avatarName, t)
 408       }
 409
 410       // Force update
 411       actor.setDataValue('updatedAt', new Date())
 412       await actor.save({ transaction: t })
 413
 414       if (actor.Account) {
 415         actor.Account.set('name', result.name)
 416         actor.Account.set('description', result.summary)
 417
 418         await actor.Account.save({ transaction: t })
 419       } else if (actor.VideoChannel) {
 420         actor.VideoChannel.set('name', result.name)
 421         actor.VideoChannel.set('description', result.summary)
 422         actor.VideoChannel.set('support', result.support)
 423
 424         await actor.VideoChannel.save({ transaction: t })
 425       }
 426
 427       return { refreshed: true, actor }
 428     })
 429   } catch (err) {
 430     logger.warn('Cannot refresh actor.', { err })
 431     return { actor, refreshed: false }
 432   }
 433 }