server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24 import { CONFIG } from '../../initializers/config'
  25 import { sequelizeTypescript } from '../../initializers/database'
  26
  27 // Set account keys, this could be long so process after the account creation and do not block the client
  28 function setAsyncActorKeys (actor: ActorModel) {
  29   return createPrivateAndPublicKeys()
  30     .then(({ publicKey, privateKey }) => {
  31       actor.set('publicKey', publicKey)
  32       actor.set('privateKey', privateKey)
  33       return actor.save()
  34     })
  35     .catch(err => {
  36       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  37       return actor
  38     })
  39 }
  40
  41 async function getOrCreateActorAndServerAndModel (
  42   activityActor: string | ActivityPubActor,
  43   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  44   recurseIfNeeded = true,
  45   updateCollections = false
  46 ) {
  47   const actorUrl = getAPId(activityActor)
  48   let created = false
  49   let accountPlaylistsUrl: string
  50
  51   let actor = await fetchActorByUrl(actorUrl, fetchType)
  52   // Orphan actor (not associated to an account of channel) so recreate it
  53   if (actor && (!actor.Account && !actor.VideoChannel)) {
  54     await actor.destroy()
  55     actor = null
  56   }
  57
  58   // We don't have this actor in our database, fetch it on remote
  59   if (!actor) {
  60     const { result } = await fetchRemoteActor(actorUrl)
  61     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  62
  63     // Create the attributed to actor
  64     // In PeerTube a video channel is owned by an account
  65     let ownerActor: ActorModel = undefined
  66     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  67       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  68       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  69
  70       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  71         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  72       }
  73
  74       try {
  75         // Don't recurse another time
  76         const recurseIfNeeded = false
  77         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
  78       } catch (err) {
  79         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  80         throw new Error(err)
  81       }
  82     }
  83
  84     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  85     created = true
  86     accountPlaylistsUrl = result.playlists
  87   }
  88
  89   if (actor.Account) actor.Account.Actor = actor
  90   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  91
  92   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  93   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  94
  95   if ((created === true || refreshed === true) && updateCollections === true) {
  96     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  97     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  98   }
  99
 100   // We created a new account: fetch the playlists
 101   if (created === true && actor.Account && accountPlaylistsUrl) {
 102     const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
 103     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 104   }
 105
 106   return actorRefreshed
 107 }
 108
 109 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 110   return new ActorModel({
 111     type,
 112     url,
 113     preferredUsername,
 114     uuid,
 115     publicKey: null,
 116     privateKey: null,
 117     followersCount: 0,
 118     followingCount: 0,
 119     inboxUrl: url + '/inbox',
 120     outboxUrl: url + '/outbox',
 121     sharedInboxUrl: WEBSERVER.URL + '/inbox',
 122     followersUrl: url + '/followers',
 123     followingUrl: url + '/following'
 124   })
 125 }
 126
 127 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 128   const followersCount = await fetchActorTotalItems(attributes.followers)
 129   const followingCount = await fetchActorTotalItems(attributes.following)
 130
 131   actorInstance.set('type', attributes.type)
 132   actorInstance.set('uuid', attributes.uuid)
 133   actorInstance.set('preferredUsername', attributes.preferredUsername)
 134   actorInstance.set('url', attributes.id)
 135   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 136   actorInstance.set('followersCount', followersCount)
 137   actorInstance.set('followingCount', followingCount)
 138   actorInstance.set('inboxUrl', attributes.inbox)
 139   actorInstance.set('outboxUrl', attributes.outbox)
 140   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 141   actorInstance.set('followersUrl', attributes.followers)
 142   actorInstance.set('followingUrl', attributes.following)
 143 }
 144
 145 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 146   if (avatarName !== undefined) {
 147     if (actorInstance.avatarId) {
 148       try {
 149         await actorInstance.Avatar.destroy({ transaction: t })
 150       } catch (err) {
 151         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 152       }
 153     }
 154
 155     const avatar = await AvatarModel.create({
 156       filename: avatarName
 157     }, { transaction: t })
 158
 159     actorInstance.set('avatarId', avatar.id)
 160     actorInstance.Avatar = avatar
 161   }
 162
 163   return actorInstance
 164 }
 165
 166 async function fetchActorTotalItems (url: string) {
 167   const options = {
 168     uri: url,
 169     method: 'GET',
 170     json: true,
 171     activityPub: true
 172   }
 173
 174   try {
 175     const { body } = await doRequest(options)
 176     return body.totalItems ? body.totalItems : 0
 177   } catch (err) {
 178     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 179     return 0
 180   }
 181 }
 182
 183 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 184   if (
 185     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 186     isActivityPubUrlValid(actorJSON.icon.url)
 187   ) {
 188     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 189
 190     const avatarName = uuidv4() + extension
 191     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 192
 193     return avatarName
 194   }
 195
 196   return undefined
 197 }
 198
 199 async function addFetchOutboxJob (actor: ActorModel) {
 200   // Don't fetch ourselves
 201   const serverActor = await getServerActor()
 202   if (serverActor.id === actor.id) {
 203     logger.error('Cannot fetch our own outbox!')
 204     return undefined
 205   }
 206
 207   const payload = {
 208     uri: actor.outboxUrl,
 209     type: 'activity' as 'activity'
 210   }
 211
 212   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 213 }
 214
 215 async function refreshActorIfNeeded (
 216   actorArg: ActorModel,
 217   fetchedType: ActorFetchByUrlType
 218 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 219   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 220
 221   // We need more attributes
 222   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 223
 224   try {
 225     let actorUrl: string
 226     try {
 227       actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 228     } catch (err) {
 229       logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
 230       actorUrl = actor.url
 231     }
 232
 233     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 234
 235     if (statusCode === 404) {
 236       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 237       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 238       return { actor: undefined, refreshed: false }
 239     }
 240
 241     if (result === undefined) {
 242       logger.warn('Cannot fetch remote actor in refresh actor.')
 243       return { actor, refreshed: false }
 244     }
 245
 246     return sequelizeTypescript.transaction(async t => {
 247       updateInstanceWithAnother(actor, result.actor)
 248
 249       if (result.avatarName !== undefined) {
 250         await updateActorAvatarInstance(actor, result.avatarName, t)
 251       }
 252
 253       // Force update
 254       actor.setDataValue('updatedAt', new Date())
 255       await actor.save({ transaction: t })
 256
 257       if (actor.Account) {
 258         actor.Account.set('name', result.name)
 259         actor.Account.set('description', result.summary)
 260
 261         await actor.Account.save({ transaction: t })
 262       } else if (actor.VideoChannel) {
 263         actor.VideoChannel.set('name', result.name)
 264         actor.VideoChannel.set('description', result.summary)
 265         actor.VideoChannel.set('support', result.support)
 266
 267         await actor.VideoChannel.save({ transaction: t })
 268       }
 269
 270       return { refreshed: true, actor }
 271     })
 272   } catch (err) {
 273     logger.warn('Cannot refresh actor.', { err })
 274     return { actor, refreshed: false }
 275   }
 276 }
 277
 278 export {
 279   getOrCreateActorAndServerAndModel,
 280   buildActorInstance,
 281   setAsyncActorKeys,
 282   fetchActorTotalItems,
 283   fetchAvatarIfExists,
 284   updateActorInstance,
 285   refreshActorIfNeeded,
 286   updateActorAvatarInstance,
 287   addFetchOutboxJob
 288 }
 289
 290 // ---------------------------------------------------------------------------
 291
 292 function saveActorAndServerAndModelIfNotExist (
 293   result: FetchRemoteActorResult,
 294   ownerActor?: ActorModel,
 295   t?: Transaction
 296 ): Bluebird<ActorModel> | Promise<ActorModel> {
 297   let actor = result.actor
 298
 299   if (t !== undefined) return save(t)
 300
 301   return sequelizeTypescript.transaction(t => save(t))
 302
 303   async function save (t: Transaction) {
 304     const actorHost = url.parse(actor.url).host
 305
 306     const serverOptions = {
 307       where: {
 308         host: actorHost
 309       },
 310       defaults: {
 311         host: actorHost
 312       },
 313       transaction: t
 314     }
 315     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 316
 317     // Save our new account in database
 318     actor.set('serverId', server.id)
 319
 320     // Avatar?
 321     if (result.avatarName) {
 322       const avatar = await AvatarModel.create({
 323         filename: result.avatarName
 324       }, { transaction: t })
 325       actor.set('avatarId', avatar.id)
 326     }
 327
 328     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 329     // (which could be false in a retried query)
 330     const [ actorCreated ] = await ActorModel.findOrCreate({
 331       defaults: actor.toJSON(),
 332       where: {
 333         url: actor.url
 334       },
 335       transaction: t
 336     })
 337
 338     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 339       actorCreated.Account = await saveAccount(actorCreated, result, t)
 340       actorCreated.Account.Actor = actorCreated
 341     } else if (actorCreated.type === 'Group') { // Video channel
 342       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 343       actorCreated.VideoChannel.Actor = actorCreated
 344       actorCreated.VideoChannel.Account = ownerActor.Account
 345     }
 346
 347     actorCreated.Server = server
 348
 349     return actorCreated
 350   }
 351 }
 352
 353 type FetchRemoteActorResult = {
 354   actor: ActorModel
 355   name: string
 356   summary: string
 357   support?: string
 358   playlists?: string
 359   avatarName?: string
 360   attributedTo: ActivityPubAttributedTo[]
 361 }
 362 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 363   const options = {
 364     uri: actorUrl,
 365     method: 'GET',
 366     json: true,
 367     activityPub: true
 368   }
 369
 370   logger.info('Fetching remote actor %s.', actorUrl)
 371
 372   const requestResult = await doRequest<ActivityPubActor>(options)
 373   normalizeActor(requestResult.body)
 374
 375   const actorJSON = requestResult.body
 376   if (isActorObjectValid(actorJSON) === false) {
 377     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 378     return { result: undefined, statusCode: requestResult.response.statusCode }
 379   }
 380
 381   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 382     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
 383     return { result: undefined, statusCode: requestResult.response.statusCode }
 384   }
 385
 386   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 387   const followingCount = await fetchActorTotalItems(actorJSON.following)
 388
 389   const actor = new ActorModel({
 390     type: actorJSON.type,
 391     uuid: actorJSON.uuid,
 392     preferredUsername: actorJSON.preferredUsername,
 393     url: actorJSON.id,
 394     publicKey: actorJSON.publicKey.publicKeyPem,
 395     privateKey: null,
 396     followersCount: followersCount,
 397     followingCount: followingCount,
 398     inboxUrl: actorJSON.inbox,
 399     outboxUrl: actorJSON.outbox,
 400     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 401     followersUrl: actorJSON.followers,
 402     followingUrl: actorJSON.following
 403   })
 404
 405   const avatarName = await fetchAvatarIfExists(actorJSON)
 406
 407   const name = actorJSON.name || actorJSON.preferredUsername
 408   return {
 409     statusCode: requestResult.response.statusCode,
 410     result: {
 411       actor,
 412       name,
 413       avatarName,
 414       summary: actorJSON.summary,
 415       support: actorJSON.support,
 416       playlists: actorJSON.playlists,
 417       attributedTo: actorJSON.attributedTo
 418     }
 419   }
 420 }
 421
 422 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 423   const [ accountCreated ] = await AccountModel.findOrCreate({
 424     defaults: {
 425       name: result.name,
 426       description: result.summary,
 427       actorId: actor.id
 428     },
 429     where: {
 430       actorId: actor.id
 431     },
 432     transaction: t
 433   })
 434
 435   return accountCreated
 436 }
 437
 438 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 439   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 440     defaults: {
 441       name: result.name,
 442       description: result.summary,
 443       support: result.support,
 444       actorId: actor.id,
 445       accountId: ownerActor.Account.id
 446     },
 447     where: {
 448       actorId: actor.id
 449     },
 450     transaction: t
 451   })
 452
 453   return videoChannelCreated
 454 }