server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24
  25 // Set account keys, this could be long so process after the account creation and do not block the client
  26 function setAsyncActorKeys (actor: ActorModel) {
  27   return createPrivateAndPublicKeys()
  28     .then(({ publicKey, privateKey }) => {
  29       actor.set('publicKey', publicKey)
  30       actor.set('privateKey', privateKey)
  31       return actor.save()
  32     })
  33     .catch(err => {
  34       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  35       return actor
  36     })
  37 }
  38
  39 async function getOrCreateActorAndServerAndModel (
  40   activityActor: string | ActivityPubActor,
  41   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  42   recurseIfNeeded = true,
  43   updateCollections = false
  44 ) {
  45   const actorUrl = getAPId(activityActor)
  46   let created = false
  47   let accountPlaylistsUrl: string
  48
  49   let actor = await fetchActorByUrl(actorUrl, fetchType)
  50   // Orphan actor (not associated to an account of channel) so recreate it
  51   if (actor && (!actor.Account && !actor.VideoChannel)) {
  52     await actor.destroy()
  53     actor = null
  54   }
  55
  56   // We don't have this actor in our database, fetch it on remote
  57   if (!actor) {
  58     const { result } = await fetchRemoteActor(actorUrl)
  59     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  60
  61     // Create the attributed to actor
  62     // In PeerTube a video channel is owned by an account
  63     let ownerActor: ActorModel = undefined
  64     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  65       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  66       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  67
  68       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  69         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  70       }
  71
  72       try {
  73         // Don't recurse another time
  74         const recurseIfNeeded = false
  75         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
  76       } catch (err) {
  77         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  78         throw new Error(err)
  79       }
  80     }
  81
  82     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  83     created = true
  84     accountPlaylistsUrl = result.playlists
  85   }
  86
  87   if (actor.Account) actor.Account.Actor = actor
  88   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  89
  90   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  91   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  92
  93   if ((created === true || refreshed === true) && updateCollections === true) {
  94     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  95     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  96   }
  97
  98   // We created a new account: fetch the playlists
  99   if (created === true && actor.Account && accountPlaylistsUrl) {
 100     const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
 101     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 102   }
 103
 104   return actorRefreshed
 105 }
 106
 107 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 108   return new ActorModel({
 109     type,
 110     url,
 111     preferredUsername,
 112     uuid,
 113     publicKey: null,
 114     privateKey: null,
 115     followersCount: 0,
 116     followingCount: 0,
 117     inboxUrl: url + '/inbox',
 118     outboxUrl: url + '/outbox',
 119     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 120     followersUrl: url + '/followers',
 121     followingUrl: url + '/following'
 122   })
 123 }
 124
 125 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 126   const followersCount = await fetchActorTotalItems(attributes.followers)
 127   const followingCount = await fetchActorTotalItems(attributes.following)
 128
 129   actorInstance.set('type', attributes.type)
 130   actorInstance.set('uuid', attributes.uuid)
 131   actorInstance.set('preferredUsername', attributes.preferredUsername)
 132   actorInstance.set('url', attributes.id)
 133   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 134   actorInstance.set('followersCount', followersCount)
 135   actorInstance.set('followingCount', followingCount)
 136   actorInstance.set('inboxUrl', attributes.inbox)
 137   actorInstance.set('outboxUrl', attributes.outbox)
 138   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 139   actorInstance.set('followersUrl', attributes.followers)
 140   actorInstance.set('followingUrl', attributes.following)
 141 }
 142
 143 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 144   if (avatarName !== undefined) {
 145     if (actorInstance.avatarId) {
 146       try {
 147         await actorInstance.Avatar.destroy({ transaction: t })
 148       } catch (err) {
 149         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 150       }
 151     }
 152
 153     const avatar = await AvatarModel.create({
 154       filename: avatarName
 155     }, { transaction: t })
 156
 157     actorInstance.set('avatarId', avatar.id)
 158     actorInstance.Avatar = avatar
 159   }
 160
 161   return actorInstance
 162 }
 163
 164 async function fetchActorTotalItems (url: string) {
 165   const options = {
 166     uri: url,
 167     method: 'GET',
 168     json: true,
 169     activityPub: true
 170   }
 171
 172   try {
 173     const { body } = await doRequest(options)
 174     return body.totalItems ? body.totalItems : 0
 175   } catch (err) {
 176     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 177     return 0
 178   }
 179 }
 180
 181 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 182   if (
 183     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 184     isActivityPubUrlValid(actorJSON.icon.url)
 185   ) {
 186     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 187
 188     const avatarName = uuidv4() + extension
 189     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 190
 191     return avatarName
 192   }
 193
 194   return undefined
 195 }
 196
 197 async function addFetchOutboxJob (actor: ActorModel) {
 198   // Don't fetch ourselves
 199   const serverActor = await getServerActor()
 200   if (serverActor.id === actor.id) {
 201     logger.error('Cannot fetch our own outbox!')
 202     return undefined
 203   }
 204
 205   const payload = {
 206     uri: actor.outboxUrl,
 207     type: 'activity' as 'activity'
 208   }
 209
 210   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 211 }
 212
 213 async function refreshActorIfNeeded (
 214   actorArg: ActorModel,
 215   fetchedType: ActorFetchByUrlType
 216 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 217   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 218
 219   // We need more attributes
 220   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 221
 222   try {
 223     let actorUrl: string
 224     try {
 225       actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 226     } catch (err) {
 227       logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
 228       actorUrl = actor.url
 229     }
 230
 231     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 232
 233     if (statusCode === 404) {
 234       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 235       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 236       return { actor: undefined, refreshed: false }
 237     }
 238
 239     if (result === undefined) {
 240       logger.warn('Cannot fetch remote actor in refresh actor.')
 241       return { actor, refreshed: false }
 242     }
 243
 244     return sequelizeTypescript.transaction(async t => {
 245       updateInstanceWithAnother(actor, result.actor)
 246
 247       if (result.avatarName !== undefined) {
 248         await updateActorAvatarInstance(actor, result.avatarName, t)
 249       }
 250
 251       // Force update
 252       actor.setDataValue('updatedAt', new Date())
 253       await actor.save({ transaction: t })
 254
 255       if (actor.Account) {
 256         actor.Account.set('name', result.name)
 257         actor.Account.set('description', result.summary)
 258
 259         await actor.Account.save({ transaction: t })
 260       } else if (actor.VideoChannel) {
 261         actor.VideoChannel.set('name', result.name)
 262         actor.VideoChannel.set('description', result.summary)
 263         actor.VideoChannel.set('support', result.support)
 264
 265         await actor.VideoChannel.save({ transaction: t })
 266       }
 267
 268       return { refreshed: true, actor }
 269     })
 270   } catch (err) {
 271     logger.warn('Cannot refresh actor.', { err })
 272     return { actor, refreshed: false }
 273   }
 274 }
 275
 276 export {
 277   getOrCreateActorAndServerAndModel,
 278   buildActorInstance,
 279   setAsyncActorKeys,
 280   fetchActorTotalItems,
 281   fetchAvatarIfExists,
 282   updateActorInstance,
 283   refreshActorIfNeeded,
 284   updateActorAvatarInstance,
 285   addFetchOutboxJob
 286 }
 287
 288 // ---------------------------------------------------------------------------
 289
 290 function saveActorAndServerAndModelIfNotExist (
 291   result: FetchRemoteActorResult,
 292   ownerActor?: ActorModel,
 293   t?: Transaction
 294 ): Bluebird<ActorModel> | Promise<ActorModel> {
 295   let actor = result.actor
 296
 297   if (t !== undefined) return save(t)
 298
 299   return sequelizeTypescript.transaction(t => save(t))
 300
 301   async function save (t: Transaction) {
 302     const actorHost = url.parse(actor.url).host
 303
 304     const serverOptions = {
 305       where: {
 306         host: actorHost
 307       },
 308       defaults: {
 309         host: actorHost
 310       },
 311       transaction: t
 312     }
 313     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 314
 315     // Save our new account in database
 316     actor.set('serverId', server.id)
 317
 318     // Avatar?
 319     if (result.avatarName) {
 320       const avatar = await AvatarModel.create({
 321         filename: result.avatarName
 322       }, { transaction: t })
 323       actor.set('avatarId', avatar.id)
 324     }
 325
 326     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 327     // (which could be false in a retried query)
 328     const [ actorCreated ] = await ActorModel.findOrCreate({
 329       defaults: actor.toJSON(),
 330       where: {
 331         url: actor.url
 332       },
 333       transaction: t
 334     })
 335
 336     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 337       actorCreated.Account = await saveAccount(actorCreated, result, t)
 338       actorCreated.Account.Actor = actorCreated
 339     } else if (actorCreated.type === 'Group') { // Video channel
 340       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 341       actorCreated.VideoChannel.Actor = actorCreated
 342       actorCreated.VideoChannel.Account = ownerActor.Account
 343     }
 344
 345     actorCreated.Server = server
 346
 347     return actorCreated
 348   }
 349 }
 350
 351 type FetchRemoteActorResult = {
 352   actor: ActorModel
 353   name: string
 354   summary: string
 355   support?: string
 356   playlists?: string
 357   avatarName?: string
 358   attributedTo: ActivityPubAttributedTo[]
 359 }
 360 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 361   const options = {
 362     uri: actorUrl,
 363     method: 'GET',
 364     json: true,
 365     activityPub: true
 366   }
 367
 368   logger.info('Fetching remote actor %s.', actorUrl)
 369
 370   const requestResult = await doRequest<ActivityPubActor>(options)
 371   normalizeActor(requestResult.body)
 372
 373   const actorJSON = requestResult.body
 374   if (isActorObjectValid(actorJSON) === false) {
 375     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 376     return { result: undefined, statusCode: requestResult.response.statusCode }
 377   }
 378
 379   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 380     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
 381     return { result: undefined, statusCode: requestResult.response.statusCode }
 382   }
 383
 384   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 385   const followingCount = await fetchActorTotalItems(actorJSON.following)
 386
 387   const actor = new ActorModel({
 388     type: actorJSON.type,
 389     uuid: actorJSON.uuid,
 390     preferredUsername: actorJSON.preferredUsername,
 391     url: actorJSON.id,
 392     publicKey: actorJSON.publicKey.publicKeyPem,
 393     privateKey: null,
 394     followersCount: followersCount,
 395     followingCount: followingCount,
 396     inboxUrl: actorJSON.inbox,
 397     outboxUrl: actorJSON.outbox,
 398     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 399     followersUrl: actorJSON.followers,
 400     followingUrl: actorJSON.following
 401   })
 402
 403   const avatarName = await fetchAvatarIfExists(actorJSON)
 404
 405   const name = actorJSON.name || actorJSON.preferredUsername
 406   return {
 407     statusCode: requestResult.response.statusCode,
 408     result: {
 409       actor,
 410       name,
 411       avatarName,
 412       summary: actorJSON.summary,
 413       support: actorJSON.support,
 414       playlists: actorJSON.playlists,
 415       attributedTo: actorJSON.attributedTo
 416     }
 417   }
 418 }
 419
 420 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 421   const [ accountCreated ] = await AccountModel.findOrCreate({
 422     defaults: {
 423       name: result.name,
 424       description: result.summary,
 425       actorId: actor.id
 426     },
 427     where: {
 428       actorId: actor.id
 429     },
 430     transaction: t
 431   })
 432
 433   return accountCreated
 434 }
 435
 436 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 437   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 438     defaults: {
 439       name: result.name,
 440       description: result.summary,
 441       support: result.support,
 442       actorId: actor.id,
 443       accountId: ownerActor.Account.id
 444     },
 445     where: {
 446       actorId: actor.id
 447     },
 448     transaction: t
 449   })
 450
 451   return videoChannelCreated
 452 }