server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
   8 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest, downloadImage } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { AVATARS_SIZE, MIMETYPES, sequelizeTypescript, WEBSERVER } from '../../initializers'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24 import { CONFIG } from '../../initializers/config'
  25
  26 // Set account keys, this could be long so process after the account creation and do not block the client
  27 function setAsyncActorKeys (actor: ActorModel) {
  28   return createPrivateAndPublicKeys()
  29     .then(({ publicKey, privateKey }) => {
  30       actor.set('publicKey', publicKey)
  31       actor.set('privateKey', privateKey)
  32       return actor.save()
  33     })
  34     .catch(err => {
  35       logger.error('Cannot set public/private keys of actor %d.', actor.uuid, { err })
  36       return actor
  37     })
  38 }
  39
  40 async function getOrCreateActorAndServerAndModel (
  41   activityActor: string | ActivityPubActor,
  42   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  43   recurseIfNeeded = true,
  44   updateCollections = false
  45 ) {
  46   const actorUrl = getAPId(activityActor)
  47   let created = false
  48   let accountPlaylistsUrl: string
  49
  50   let actor = await fetchActorByUrl(actorUrl, fetchType)
  51   // Orphan actor (not associated to an account of channel) so recreate it
  52   if (actor && (!actor.Account && !actor.VideoChannel)) {
  53     await actor.destroy()
  54     actor = null
  55   }
  56
  57   // We don't have this actor in our database, fetch it on remote
  58   if (!actor) {
  59     const { result } = await fetchRemoteActor(actorUrl)
  60     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  61
  62     // Create the attributed to actor
  63     // In PeerTube a video channel is owned by an account
  64     let ownerActor: ActorModel = undefined
  65     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  66       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  67       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  68
  69       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  70         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  71       }
  72
  73       try {
  74         // Don't recurse another time
  75         const recurseIfNeeded = false
  76         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
  77       } catch (err) {
  78         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  79         throw new Error(err)
  80       }
  81     }
  82
  83     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  84     created = true
  85     accountPlaylistsUrl = result.playlists
  86   }
  87
  88   if (actor.Account) actor.Account.Actor = actor
  89   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  90
  91   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  92   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  93
  94   if ((created === true || refreshed === true) && updateCollections === true) {
  95     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  96     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  97   }
  98
  99   // We created a new account: fetch the playlists
 100   if (created === true && actor.Account && accountPlaylistsUrl) {
 101     const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
 102     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 103   }
 104
 105   return actorRefreshed
 106 }
 107
 108 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 109   return new ActorModel({
 110     type,
 111     url,
 112     preferredUsername,
 113     uuid,
 114     publicKey: null,
 115     privateKey: null,
 116     followersCount: 0,
 117     followingCount: 0,
 118     inboxUrl: url + '/inbox',
 119     outboxUrl: url + '/outbox',
 120     sharedInboxUrl: WEBSERVER.URL + '/inbox',
 121     followersUrl: url + '/followers',
 122     followingUrl: url + '/following'
 123   })
 124 }
 125
 126 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 127   const followersCount = await fetchActorTotalItems(attributes.followers)
 128   const followingCount = await fetchActorTotalItems(attributes.following)
 129
 130   actorInstance.set('type', attributes.type)
 131   actorInstance.set('uuid', attributes.uuid)
 132   actorInstance.set('preferredUsername', attributes.preferredUsername)
 133   actorInstance.set('url', attributes.id)
 134   actorInstance.set('publicKey', attributes.publicKey.publicKeyPem)
 135   actorInstance.set('followersCount', followersCount)
 136   actorInstance.set('followingCount', followingCount)
 137   actorInstance.set('inboxUrl', attributes.inbox)
 138   actorInstance.set('outboxUrl', attributes.outbox)
 139   actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox)
 140   actorInstance.set('followersUrl', attributes.followers)
 141   actorInstance.set('followingUrl', attributes.following)
 142 }
 143
 144 async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) {
 145   if (avatarName !== undefined) {
 146     if (actorInstance.avatarId) {
 147       try {
 148         await actorInstance.Avatar.destroy({ transaction: t })
 149       } catch (err) {
 150         logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err })
 151       }
 152     }
 153
 154     const avatar = await AvatarModel.create({
 155       filename: avatarName
 156     }, { transaction: t })
 157
 158     actorInstance.set('avatarId', avatar.id)
 159     actorInstance.Avatar = avatar
 160   }
 161
 162   return actorInstance
 163 }
 164
 165 async function fetchActorTotalItems (url: string) {
 166   const options = {
 167     uri: url,
 168     method: 'GET',
 169     json: true,
 170     activityPub: true
 171   }
 172
 173   try {
 174     const { body } = await doRequest(options)
 175     return body.totalItems ? body.totalItems : 0
 176   } catch (err) {
 177     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 178     return 0
 179   }
 180 }
 181
 182 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
 183   if (
 184     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 185     isActivityPubUrlValid(actorJSON.icon.url)
 186   ) {
 187     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 188
 189     const avatarName = uuidv4() + extension
 190     await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 191
 192     return avatarName
 193   }
 194
 195   return undefined
 196 }
 197
 198 async function addFetchOutboxJob (actor: ActorModel) {
 199   // Don't fetch ourselves
 200   const serverActor = await getServerActor()
 201   if (serverActor.id === actor.id) {
 202     logger.error('Cannot fetch our own outbox!')
 203     return undefined
 204   }
 205
 206   const payload = {
 207     uri: actor.outboxUrl,
 208     type: 'activity' as 'activity'
 209   }
 210
 211   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 212 }
 213
 214 async function refreshActorIfNeeded (
 215   actorArg: ActorModel,
 216   fetchedType: ActorFetchByUrlType
 217 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 218   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 219
 220   // We need more attributes
 221   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 222
 223   try {
 224     let actorUrl: string
 225     try {
 226       actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 227     } catch (err) {
 228       logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
 229       actorUrl = actor.url
 230     }
 231
 232     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 233
 234     if (statusCode === 404) {
 235       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 236       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 237       return { actor: undefined, refreshed: false }
 238     }
 239
 240     if (result === undefined) {
 241       logger.warn('Cannot fetch remote actor in refresh actor.')
 242       return { actor, refreshed: false }
 243     }
 244
 245     return sequelizeTypescript.transaction(async t => {
 246       updateInstanceWithAnother(actor, result.actor)
 247
 248       if (result.avatarName !== undefined) {
 249         await updateActorAvatarInstance(actor, result.avatarName, t)
 250       }
 251
 252       // Force update
 253       actor.setDataValue('updatedAt', new Date())
 254       await actor.save({ transaction: t })
 255
 256       if (actor.Account) {
 257         actor.Account.set('name', result.name)
 258         actor.Account.set('description', result.summary)
 259
 260         await actor.Account.save({ transaction: t })
 261       } else if (actor.VideoChannel) {
 262         actor.VideoChannel.set('name', result.name)
 263         actor.VideoChannel.set('description', result.summary)
 264         actor.VideoChannel.set('support', result.support)
 265
 266         await actor.VideoChannel.save({ transaction: t })
 267       }
 268
 269       return { refreshed: true, actor }
 270     })
 271   } catch (err) {
 272     logger.warn('Cannot refresh actor.', { err })
 273     return { actor, refreshed: false }
 274   }
 275 }
 276
 277 export {
 278   getOrCreateActorAndServerAndModel,
 279   buildActorInstance,
 280   setAsyncActorKeys,
 281   fetchActorTotalItems,
 282   fetchAvatarIfExists,
 283   updateActorInstance,
 284   refreshActorIfNeeded,
 285   updateActorAvatarInstance,
 286   addFetchOutboxJob
 287 }
 288
 289 // ---------------------------------------------------------------------------
 290
 291 function saveActorAndServerAndModelIfNotExist (
 292   result: FetchRemoteActorResult,
 293   ownerActor?: ActorModel,
 294   t?: Transaction
 295 ): Bluebird<ActorModel> | Promise<ActorModel> {
 296   let actor = result.actor
 297
 298   if (t !== undefined) return save(t)
 299
 300   return sequelizeTypescript.transaction(t => save(t))
 301
 302   async function save (t: Transaction) {
 303     const actorHost = url.parse(actor.url).host
 304
 305     const serverOptions = {
 306       where: {
 307         host: actorHost
 308       },
 309       defaults: {
 310         host: actorHost
 311       },
 312       transaction: t
 313     }
 314     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 315
 316     // Save our new account in database
 317     actor.set('serverId', server.id)
 318
 319     // Avatar?
 320     if (result.avatarName) {
 321       const avatar = await AvatarModel.create({
 322         filename: result.avatarName
 323       }, { transaction: t })
 324       actor.set('avatarId', avatar.id)
 325     }
 326
 327     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 328     // (which could be false in a retried query)
 329     const [ actorCreated ] = await ActorModel.findOrCreate({
 330       defaults: actor.toJSON(),
 331       where: {
 332         url: actor.url
 333       },
 334       transaction: t
 335     })
 336
 337     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 338       actorCreated.Account = await saveAccount(actorCreated, result, t)
 339       actorCreated.Account.Actor = actorCreated
 340     } else if (actorCreated.type === 'Group') { // Video channel
 341       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 342       actorCreated.VideoChannel.Actor = actorCreated
 343       actorCreated.VideoChannel.Account = ownerActor.Account
 344     }
 345
 346     actorCreated.Server = server
 347
 348     return actorCreated
 349   }
 350 }
 351
 352 type FetchRemoteActorResult = {
 353   actor: ActorModel
 354   name: string
 355   summary: string
 356   support?: string
 357   playlists?: string
 358   avatarName?: string
 359   attributedTo: ActivityPubAttributedTo[]
 360 }
 361 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 362   const options = {
 363     uri: actorUrl,
 364     method: 'GET',
 365     json: true,
 366     activityPub: true
 367   }
 368
 369   logger.info('Fetching remote actor %s.', actorUrl)
 370
 371   const requestResult = await doRequest<ActivityPubActor>(options)
 372   normalizeActor(requestResult.body)
 373
 374   const actorJSON = requestResult.body
 375   if (isActorObjectValid(actorJSON) === false) {
 376     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 377     return { result: undefined, statusCode: requestResult.response.statusCode }
 378   }
 379
 380   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 381     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
 382     return { result: undefined, statusCode: requestResult.response.statusCode }
 383   }
 384
 385   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 386   const followingCount = await fetchActorTotalItems(actorJSON.following)
 387
 388   const actor = new ActorModel({
 389     type: actorJSON.type,
 390     uuid: actorJSON.uuid,
 391     preferredUsername: actorJSON.preferredUsername,
 392     url: actorJSON.id,
 393     publicKey: actorJSON.publicKey.publicKeyPem,
 394     privateKey: null,
 395     followersCount: followersCount,
 396     followingCount: followingCount,
 397     inboxUrl: actorJSON.inbox,
 398     outboxUrl: actorJSON.outbox,
 399     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 400     followersUrl: actorJSON.followers,
 401     followingUrl: actorJSON.following
 402   })
 403
 404   const avatarName = await fetchAvatarIfExists(actorJSON)
 405
 406   const name = actorJSON.name || actorJSON.preferredUsername
 407   return {
 408     statusCode: requestResult.response.statusCode,
 409     result: {
 410       actor,
 411       name,
 412       avatarName,
 413       summary: actorJSON.summary,
 414       support: actorJSON.support,
 415       playlists: actorJSON.playlists,
 416       attributedTo: actorJSON.attributedTo
 417     }
 418   }
 419 }
 420
 421 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 422   const [ accountCreated ] = await AccountModel.findOrCreate({
 423     defaults: {
 424       name: result.name,
 425       description: result.summary,
 426       actorId: actor.id
 427     },
 428     where: {
 429       actorId: actor.id
 430     },
 431     transaction: t
 432   })
 433
 434   return accountCreated
 435 }
 436
 437 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 438   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 439     defaults: {
 440       name: result.name,
 441       description: result.summary,
 442       support: result.support,
 443       actorId: actor.id,
 444       accountId: ownerActor.Account.id
 445     },
 446     where: {
 447       actorId: actor.id
 448     },
 449     transaction: t
 450   })
 451
 452   return videoChannelCreated
 453 }