server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, CONFIG, MIMETYPES, sequelizeTypescript } from '../../initializers'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24
  25 // Set account keys, this could be long so process after the account creation and do not block the client
  26 function setAsyncActorKeys (actor: ActorModel) {
  27   return createPrivateAndPublicKeys()
  28     .then(({ publicKey, privateKey }) => {
  29       actor.set('publicKey', publicKey)
  30       actor.set('privateKey', privateKey)
  31       return actor.save()
  32     })
  33     .catch(err => {
  34       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  35       return actor
  36     })
  37 }
  38
  39 async function getOrCreateActorAndServerAndModel (
  40   activityActor: string | ActivityPubActor,
  41   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  42   recurseIfNeeded = true,
  43   updateCollections = false
  44 ) {
  45   const actorUrl = getAPId(activityActor)
  46   let created = false
  47   let accountPlaylistsUrl: string
  48
  49   let actor = await fetchActorByUrl(actorUrl, fetchType)
  50   // Orphan actor (not associated to an account of channel) so recreate it
  51   if (actor && (!actor.Account && !actor.VideoChannel)) {
  52     await actor.destroy()
  53     actor = null
  54   }
  55
  56   // We don't have this actor in our database, fetch it on remote
  57   if (!actor) {
  58     const { result } = await fetchRemoteActor(actorUrl)
  59     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  60
  61     // Create the attributed to actor
  62     // In PeerTube a video channel is owned by an account
  63     let ownerActor: ActorModel = undefined
  64     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  65       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  66       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  67
  68       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  69         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  70       }
  71
  72       try {
  73         // Don't recurse another time
  74         const recurseIfNeeded = false
  75         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
  76       } catch (err) {
  77         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  78         throw new Error(err)
  79       }
  80     }
  81
  82     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  83     created = true
  84     accountPlaylistsUrl = result.playlists
  85   }
  86
  87   if (actor.Account) actor.Account.Actor = actor
  88   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  89
  90   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  91   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  92
  93   if ((created === true || refreshed === true) && updateCollections === true) {
  94     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  95     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  96   }
  97
  98   // We created a new account: fetch the playlists
  99   if (created === true && actor.Account && accountPlaylistsUrl) {
 100     const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
 101     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 102   }
 103
 104   return actorRefreshed
 105 }
 106
 107 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 108   return new ActorModel({
 109     type,
 110     url,
 111     preferredUsername,
 112     uuid,
 113     publicKey: null,
 114     privateKey: null,
 115     followersCount: 0,
 116     followingCount: 0,
 117     inboxUrl: url + '/inbox',
 118     outboxUrl: url + '/outbox',
 119     sharedInboxUrl: CONFIG.WEBSERVER.URL + '/inbox',
 120     followersUrl: url + '/followers',
 121     followingUrl: url + '/following'
 122   })
 123 }
 124
 125 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 126   const followersCount = await fetchActorTotalItems(attributes.followers)
 127   const followingCount = await fetchActorTotalItems(attributes.following)
 128
 129   actorInstance.set('type', attributes.type)
 130   actorInstance.set('uuid', attributes.uuid)
 131   actorInstance.set('preferredUsername', attributes.preferredUsername)
 132   actorInstance.set('url', attributes.id)
 133   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 134   actorInstance.set('followersCount', followersCount)
 135   actorInstance.set('followingCount', followingCount)
 136   actorInstance.set('inboxUrl', attributes.inbox)
 137   actorInstance.set('outboxUrl', attributes.outbox)
 138   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 139   actorInstance.set('followersUrl', attributes.followers)
 140   actorInstance.set('followingUrl', attributes.following)
 141 }
 142
 143 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 144   if (avatarName !== undefined) {
 145     if (actorInstance.avatarId) {
 146       try {
 147         await actorInstance.Avatar.destroy({ transaction: t })
 148       } catch (err) {
 149         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 150       }
 151     }
 152
 153     const avatar = await AvatarModel.create({
 154       filename: avatarName
 155     }, { transaction: t })
 156
 157     actorInstance.set('avatarId', avatar.id)
 158     actorInstance.Avatar = avatar
 159   }
 160
 161   return actorInstance
 162 }
 163
 164 async function fetchActorTotalItems (url: string) {
 165   const options = {
 166     uri: url,
 167     method: 'GET',
 168     json: true,
 169     activityPub: true
 170   }
 171
 172   try {
 173     const { body } = await doRequest(options)
 174     return body.totalItems ? body.totalItems : 0
 175   } catch (err) {
 176     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 177     return 0
 178   }
 179 }
 180
 181 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 182   if (
 183     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 184     isActivityPubUrlValid(actorJSON.icon.url)
 185   ) {
 186     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 187
 188     const avatarName = uuidv4() + extension
 189     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 190
 191     return avatarName
 192   }
 193
 194   return undefined
 195 }
 196
 197 async function addFetchOutboxJob (actor: ActorModel) {
 198   // Don't fetch ourselves
 199   const serverActor = await getServerActor()
 200   if (serverActor.id === actor.id) {
 201     logger.error('Cannot fetch our own outbox!')
 202     return undefined
 203   }
 204
 205   const payload = {
 206     uri: actor.outboxUrl,
 207     type: 'activity' as 'activity'
 208   }
 209
 210   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 211 }
 212
 213 async function refreshActorIfNeeded (
 214   actorArg: ActorModel,
 215   fetchedType: ActorFetchByUrlType
 216 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 217   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 218
 219   // We need more attributes
 220   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 221
 222   try {
 223     let actorUrl: string
 224     try {
 225       actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 226     } catch (err) {
 227       logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
 228       actorUrl = actor.url
 229     }
 230
 231     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 232
 233     if (statusCode === 404) {
 234       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 235       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 236       return { actor: undefined, refreshed: false }
 237     }
 238
 239     if (result === undefined) {
 240       logger.warn('Cannot fetch remote actor in refresh actor.')
 241       return { actor, refreshed: false }
 242     }
 243
 244     return sequelizeTypescript.transaction(async t => {
 245       updateInstanceWithAnother(actor, result.actor)
 246
 247       if (result.avatarName !== undefined) {
 248         await updateActorAvatarInstance(actor, result.avatarName, t)
 249       }
 250
 251       // Force update
 252       actor.setDataValue('updatedAt', new Date())
 253       await actor.save({ transaction: t })
 254
 255       if (actor.Account) {
 256         actor.Account.set('name', result.name)
 257         actor.Account.set('description', result.summary)
 258
 259         await actor.Account.save({ transaction: t })
 260       } else if (actor.VideoChannel) {
 261         actor.VideoChannel.set('name', result.name)
 262         actor.VideoChannel.set('description', result.summary)
 263         actor.VideoChannel.set('support', result.support)
 264
 265         await actor.VideoChannel.save({ transaction: t })
 266       }
 267
 268       return { refreshed: true, actor }
 269     })
 270   } catch (err) {
 271     logger.warn('Cannot refresh actor.', { err })
 272     return { actor, refreshed: false }
 273   }
 274 }
 275
 276 export {
 277   getOrCreateActorAndServerAndModel,
 278   buildActorInstance,
 279   setAsyncActorKeys,
 280   fetchActorTotalItems,
 281   fetchAvatarIfExists,
 282   updateActorInstance,
 283   refreshActorIfNeeded,
 284   updateActorAvatarInstance,
 285   addFetchOutboxJob
 286 }
 287
 288 // ---------------------------------------------------------------------------
 289
 290 function saveActorAndServerAndModelIfNotExist (
 291   result: FetchRemoteActorResult,
 292   ownerActor?: ActorModel,
 293   t?: Transaction
 294 ): Bluebird<ActorModel> | Promise<ActorModel> {
 295   let actor = result.actor
 296
 297   if (t !== undefined) return save(t)
 298
 299   return sequelizeTypescript.transaction(t => save(t))
 300
 301   async function save (t: Transaction) {
 302     const actorHost = url.parse(actor.url).host
 303
 304     const serverOptions = {
 305       where: {
 306         host: actorHost
 307       },
 308       defaults: {
 309         host: actorHost
 310       },
 311       transaction: t
 312     }
 313     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 314
 315     // Save our new account in database
 316     actor.set('serverId', server.id)
 317
 318     // Avatar?
 319     if (result.avatarName) {
 320       const avatar = await AvatarModel.create({
 321         filename: result.avatarName
 322       }, { transaction: t })
 323       actor.set('avatarId', avatar.id)
 324     }
 325
 326     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 327     // (which could be false in a retried query)
 328     const [ actorCreated ] = await ActorModel.findOrCreate({
 329       defaults: actor.toJSON(),
 330       where: {
 331         url: actor.url
 332       },
 333       transaction: t
 334     })
 335
 336     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 337       actorCreated.Account = await saveAccount(actorCreated, result, t)
 338       actorCreated.Account.Actor = actorCreated
 339     } else if (actorCreated.type === 'Group') { // Video channel
 340       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 341       actorCreated.VideoChannel.Actor = actorCreated
 342       actorCreated.VideoChannel.Account = ownerActor.Account
 343     }
 344
 345     return actorCreated
 346   }
 347 }
 348
 349 type FetchRemoteActorResult = {
 350   actor: ActorModel
 351   name: string
 352   summary: string
 353   support?: string
 354   playlists?: string
 355   avatarName?: string
 356   attributedTo: ActivityPubAttributedTo[]
 357 }
 358 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 359   const options = {
 360     uri: actorUrl,
 361     method: 'GET',
 362     json: true,
 363     activityPub: true
 364   }
 365
 366   logger.info('Fetching remote actor %s.', actorUrl)
 367
 368   const requestResult = await doRequest<ActivityPubActor>(options)
 369   normalizeActor(requestResult.body)
 370
 371   const actorJSON = requestResult.body
 372   if (isActorObjectValid(actorJSON) === false) {
 373     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 374     return { result: undefined, statusCode: requestResult.response.statusCode }
 375   }
 376
 377   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 378     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
 379     return { result: undefined, statusCode: requestResult.response.statusCode }
 380   }
 381
 382   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 383   const followingCount = await fetchActorTotalItems(actorJSON.following)
 384
 385   const actor = new ActorModel({
 386     type: actorJSON.type,
 387     uuid: actorJSON.uuid,
 388     preferredUsername: actorJSON.preferredUsername,
 389     url: actorJSON.id,
 390     publicKey: actorJSON.publicKey.publicKeyPem,
 391     privateKey: null,
 392     followersCount: followersCount,
 393     followingCount: followingCount,
 394     inboxUrl: actorJSON.inbox,
 395     outboxUrl: actorJSON.outbox,
 396     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 397     followersUrl: actorJSON.followers,
 398     followingUrl: actorJSON.following
 399   })
 400
 401   const avatarName = await fetchAvatarIfExists(actorJSON)
 402
 403   const name = actorJSON.name || actorJSON.preferredUsername
 404   return {
 405     statusCode: requestResult.response.statusCode,
 406     result: {
 407       actor,
 408       name,
 409       avatarName,
 410       summary: actorJSON.summary,
 411       support: actorJSON.support,
 412       playlists: actorJSON.playlists,
 413       attributedTo: actorJSON.attributedTo
 414     }
 415   }
 416 }
 417
 418 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 419   const [ accountCreated ] = await AccountModel.findOrCreate({
 420     defaults: {
 421       name: result.name,
 422       description: result.summary,
 423       actorId: actor.id
 424     },
 425     where: {
 426       actorId: actor.id
 427     },
 428     transaction: t
 429   })
 430
 431   return accountCreated
 432 }
 433
 434 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 435   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 436     defaults: {
 437       name: result.name,
 438       description: result.summary,
 439       support: result.support,
 440       actorId: actor.id,
 441       accountId: ownerActor.Account.id
 442     },
 443     where: {
 444       actorId: actor.id
 445     },
 446     transaction: t
 447   })
 448
 449   return videoChannelCreated
 450 }