server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24
  25 // Set account keys, this could be long so process after the account creation and do not block the client
  26 function setAsyncActorKeys (actor: ActorModel) {
  27   return createPrivateAndPublicKeys()
  28     .then(({ publicKey, privateKey }) => {
  29       actor.set('publicKey', publicKey)
  30       actor.set('privateKey', privateKey)
  31       return actor.save()
  32     })
  33     .catch(err => {
  34       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  35       return actor
  36     })
  37 }
  38
  39 async function getOrCreateActorAndServerAndModel (
  40   activityActor: string | ActivityPubActor,
  41   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  42   recurseIfNeeded = true,
  43   updateCollections = false
  44 ) {
  45   const actorUrl = getAPId(activityActor)
  46   let created = false
  47
  48   let actor = await fetchActorByUrl(actorUrl, fetchType)
  49   // Orphan actor (not associated to an account of channel) so recreate it
  50   if (actor && (!actor.Account && !actor.VideoChannel)) {
  51     await actor.destroy()
  52     actor = null
  53   }
  54
  55   // We don't have this actor in our database, fetch it on remote
  56   if (!actor) {
  57     const { result } = await fetchRemoteActor(actorUrl)
  58     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  59
  60     // Create the attributed to actor
  61     // In PeerTube a video channel is owned by an account
  62     let ownerActor: ActorModel = undefined
  63     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  64       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  65       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  66
  67       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  68         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  69       }
  70
  71       try {
  72         // Don't recurse another time
  73         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', false)
  74       } catch (err) {
  75         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  76         throw new Error(err)
  77       }
  78     }
  79
  80     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  81     created = true
  82   }
  83
  84   if (actor.Account) actor.Account.Actor = actor
  85   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  86
  87   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  88   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  89
  90   if ((created === true || refreshed === true) && updateCollections === true) {
  91     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  92     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  93   }
  94
  95   return actorRefreshed
  96 }
  97
  98 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
  99   return new ActorModel({
 100     type,
 101     url,
 102     preferredUsername,
 103     uuid,
 104     publicKey: null,
 105     privateKey: null,
 106     followersCount: 0,
 107     followingCount: 0,
 108     inboxUrl: url + '/inbox',
 109     outboxUrl: url + '/outbox',
 110     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 111     followersUrl: url + '/followers',
 112     followingUrl: url + '/following'
 113   })
 114 }
 115
 116 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 117   const followersCount = await fetchActorTotalItems(attributes.followers)
 118   const followingCount = await fetchActorTotalItems(attributes.following)
 119
 120   actorInstance.set('type', attributes.type)
 121   actorInstance.set('uuid', attributes.uuid)
 122   actorInstance.set('preferredUsername', attributes.preferredUsername)
 123   actorInstance.set('url', attributes.id)
 124   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 125   actorInstance.set('followersCount', followersCount)
 126   actorInstance.set('followingCount', followingCount)
 127   actorInstance.set('inboxUrl', attributes.inbox)
 128   actorInstance.set('outboxUrl', attributes.outbox)
 129   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 130   actorInstance.set('followersUrl', attributes.followers)
 131   actorInstance.set('followingUrl', attributes.following)
 132 }
 133
 134 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 135   if (avatarName !== undefined) {
 136     if (actorInstance.avatarId) {
 137       try {
 138         await actorInstance.Avatar.destroy({ transaction: t })
 139       } catch (err) {
 140         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 141       }
 142     }
 143
 144     const avatar = await AvatarModel.create({
 145       filename: avatarName
 146     }, { transaction: t })
 147
 148     actorInstance.set('avatarId', avatar.id)
 149     actorInstance.Avatar = avatar
 150   }
 151
 152   return actorInstance
 153 }
 154
 155 async function fetchActorTotalItems (url: string) {
 156   const options = {
 157     uri: url,
 158     method: 'GET',
 159     json: true,
 160     activityPub: true
 161   }
 162
 163   try {
 164     const { body } = await doRequest(options)
 165     return body.totalItems ? body.totalItems : 0
 166   } catch (err) {
 167     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 168     return 0
 169   }
 170 }
 171
 172 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 173   if (
 174     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 175     isActivityPubUrlValid(actorJSON.icon.url)
 176   ) {
 177     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 178
 179     const avatarName = uuidv4() + extension
 180     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 181
 182     return avatarName
 183   }
 184
 185   return undefined
 186 }
 187
 188 async function addFetchOutboxJob (actor: ActorModel) {
 189   // Don't fetch ourselves
 190   const serverActor = await getServerActor()
 191   if (serverActor.id === actor.id) {
 192     logger.error('Cannot fetch our own outbox!')
 193     return undefined
 194   }
 195
 196   const payload = {
 197     uri: actor.outboxUrl,
 198     type: 'activity' as 'activity'
 199   }
 200
 201   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 202 }
 203
 204 async function refreshActorIfNeeded (
 205   actorArg: ActorModel,
 206   fetchedType: ActorFetchByUrlType
 207 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 208   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 209
 210   // We need more attributes
 211   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 212
 213   try {
 214     let actorUrl: string
 215     try {
 216       actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 217     } catch (err) {
 218       logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
 219       actorUrl = actor.url
 220     }
 221
 222     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 223
 224     if (statusCode === 404) {
 225       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 226       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 227       return { actor: undefined, refreshed: false }
 228     }
 229
 230     if (result === undefined) {
 231       logger.warn('Cannot fetch remote actor in refresh actor.')
 232       return { actor, refreshed: false }
 233     }
 234
 235     return sequelizeTypescript.transaction(async t => {
 236       updateInstanceWithAnother(actor, result.actor)
 237
 238       if (result.avatarName !== undefined) {
 239         await updateActorAvatarInstance(actor, result.avatarName, t)
 240       }
 241
 242       // Force update
 243       actor.setDataValue('updatedAt', new Date())
 244       await actor.save({ transaction: t })
 245
 246       if (actor.Account) {
 247         actor.Account.set('name', result.name)
 248         actor.Account.set('description', result.summary)
 249
 250         await actor.Account.save({ transaction: t })
 251       } else if (actor.VideoChannel) {
 252         actor.VideoChannel.set('name', result.name)
 253         actor.VideoChannel.set('description', result.summary)
 254         actor.VideoChannel.set('support', result.support)
 255
 256         await actor.VideoChannel.save({ transaction: t })
 257       }
 258
 259       return { refreshed: true, actor }
 260     })
 261   } catch (err) {
 262     logger.warn('Cannot refresh actor.', { err })
 263     return { actor, refreshed: false }
 264   }
 265 }
 266
 267 export {
 268   getOrCreateActorAndServerAndModel,
 269   buildActorInstance,
 270   setAsyncActorKeys,
 271   fetchActorTotalItems,
 272   fetchAvatarIfExists,
 273   updateActorInstance,
 274   refreshActorIfNeeded,
 275   updateActorAvatarInstance,
 276   addFetchOutboxJob
 277 }
 278
 279 // ---------------------------------------------------------------------------
 280
 281 function saveActorAndServerAndModelIfNotExist (
 282   result: FetchRemoteActorResult,
 283   ownerActor?: ActorModel,
 284   t?: Transaction
 285 ): Bluebird<ActorModel> | Promise<ActorModel> {
 286   let actor = result.actor
 287
 288   if (t !== undefined) return save(t)
 289
 290   return sequelizeTypescript.transaction(t => save(t))
 291
 292   async function save (t: Transaction) {
 293     const actorHost = url.parse(actor.url).host
 294
 295     const serverOptions = {
 296       where: {
 297         host: actorHost
 298       },
 299       defaults: {
 300         host: actorHost
 301       },
 302       transaction: t
 303     }
 304     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 305
 306     // Save our new account in database
 307     actor.set('serverId', server.id)
 308
 309     // Avatar?
 310     if (result.avatarName) {
 311       const avatar = await AvatarModel.create({
 312         filename: result.avatarName
 313       }, { transaction: t })
 314       actor.set('avatarId', avatar.id)
 315     }
 316
 317     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 318     // (which could be false in a retried query)
 319     const [ actorCreated ] = await ActorModel.findOrCreate({
 320       defaults: actor.toJSON(),
 321       where: {
 322         url: actor.url
 323       },
 324       transaction: t
 325     })
 326
 327     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 328       actorCreated.Account = await saveAccount(actorCreated, result, t)
 329       actorCreated.Account.Actor = actorCreated
 330     } else if (actorCreated.type === 'Group') { // Video channel
 331       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 332       actorCreated.VideoChannel.Actor = actorCreated
 333       actorCreated.VideoChannel.Account = ownerActor.Account
 334     }
 335
 336     return actorCreated
 337   }
 338 }
 339
 340 type FetchRemoteActorResult = {
 341   actor: ActorModel
 342   name: string
 343   summary: string
 344   support?: string
 345   avatarName?: string
 346   attributedTo: ActivityPubAttributedTo[]
 347 }
 348 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 349   const options = {
 350     uri: actorUrl,
 351     method: 'GET',
 352     json: true,
 353     activityPub: true
 354   }
 355
 356   logger.info('Fetching remote actor %s.', actorUrl)
 357
 358   const requestResult = await doRequest<ActivityPubActor>(options)
 359   normalizeActor(requestResult.body)
 360
 361   const actorJSON = requestResult.body
 362   if (isActorObjectValid(actorJSON) === false) {
 363     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 364     return { result: undefined, statusCode: requestResult.response.statusCode }
 365   }
 366
 367   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 368     throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
 369   }
 370
 371   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 372   const followingCount = await fetchActorTotalItems(actorJSON.following)
 373
 374   const actor = new ActorModel({
 375     type: actorJSON.type,
 376     uuid: actorJSON.uuid,
 377     preferredUsername: actorJSON.preferredUsername,
 378     url: actorJSON.id,
 379     publicKey: actorJSON.publicKey.publicKeyPem,
 380     privateKey: null,
 381     followersCount: followersCount,
 382     followingCount: followingCount,
 383     inboxUrl: actorJSON.inbox,
 384     outboxUrl: actorJSON.outbox,
 385     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 386     followersUrl: actorJSON.followers,
 387     followingUrl: actorJSON.following
 388   })
 389
 390   const avatarName = await fetchAvatarIfExists(actorJSON)
 391
 392   const name = actorJSON.name || actorJSON.preferredUsername
 393   return {
 394     statusCode: requestResult.response.statusCode,
 395     result: {
 396       actor,
 397       name,
 398       avatarName,
 399       summary: actorJSON.summary,
 400       support: actorJSON.support,
 401       attributedTo: actorJSON.attributedTo
 402     }
 403   }
 404 }
 405
 406 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 407   const [ accountCreated ] = await AccountModel.findOrCreate({
 408     defaults: {
 409       name: result.name,
 410       description: result.summary,
 411       actorId: actor.id
 412     },
 413     where: {
 414       actorId: actor.id
 415     },
 416     transaction: t
 417   })
 418
 419   return accountCreated
 420 }
 421
 422 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 423   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 424     defaults: {
 425       name: result.name,
 426       description: result.summary,
 427       support: result.support,
 428       actorId: actor.id,
 429       accountId: ownerActor.Account.id
 430     },
 431     where: {
 432       actorId: actor.id
 433     },
 434     transaction: t
 435   })
 436
 437   return videoChannelCreated
 438 }