server/lib/activitypub/actor.ts

   1 import * as Bluebird from 'bluebird'
   2 import { Transaction } from 'sequelize'
   3 import * as url from 'url'
   4 import * as uuidv4 from 'uuid/v4'
   5 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
   6 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
   7 import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
   8 import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor'
   9 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
  10 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
  11 import { logger } from '../../helpers/logger'
  12 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
  13 import { doRequest } from '../../helpers/requests'
  14 import { getUrlFromWebfinger } from '../../helpers/webfinger'
  15 import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
  16 import { AccountModel } from '../../models/account/account'
  17 import { ActorModel } from '../../models/activitypub/actor'
  18 import { AvatarModel } from '../../models/avatar/avatar'
  19 import { ServerModel } from '../../models/server/server'
  20 import { VideoChannelModel } from '../../models/video/video-channel'
  21 import { JobQueue } from '../job-queue'
  22 import { getServerActor } from '../../helpers/utils'
  23 import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
  24 import { sequelizeTypescript } from '../../initializers/database'
  25
  26 // Set account keys, this could be long so process after the account creation and do not block the client
  27 function setAsyncActorKeys (actor: ActorModel) {
  28   return createPrivateAndPublicKeys()
  29     .then(({ publicKey, privateKey }) => {
  30       actor.set('publicKey', publicKey)
  31       actor.set('privateKey', privateKey)
  32       return actor.save()
  33     })
  34     .catch(err => {
  35       logger.error('Cannot set public/private keys of actor %d.', actor.url, { err })
  36       return actor
  37     })
  38 }
  39
  40 async function getOrCreateActorAndServerAndModel (
  41   activityActor: string | ActivityPubActor,
  42   fetchType: ActorFetchByUrlType = 'actor-and-association-ids',
  43   recurseIfNeeded = true,
  44   updateCollections = false
  45 ) {
  46   const actorUrl = getAPId(activityActor)
  47   let created = false
  48   let accountPlaylistsUrl: string
  49
  50   let actor = await fetchActorByUrl(actorUrl, fetchType)
  51   // Orphan actor (not associated to an account of channel) so recreate it
  52   if (actor && (!actor.Account && !actor.VideoChannel)) {
  53     await actor.destroy()
  54     actor = null
  55   }
  56
  57   // We don't have this actor in our database, fetch it on remote
  58   if (!actor) {
  59     const { result } = await fetchRemoteActor(actorUrl)
  60     if (result === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl)
  61
  62     // Create the attributed to actor
  63     // In PeerTube a video channel is owned by an account
  64     let ownerActor: ActorModel = undefined
  65     if (recurseIfNeeded === true && result.actor.type === 'Group') {
  66       const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person')
  67       if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url)
  68
  69       if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) {
  70         throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`)
  71       }
  72
  73       try {
  74         // Don't recurse another time
  75         const recurseIfNeeded = false
  76         ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded)
  77       } catch (err) {
  78         logger.error('Cannot get or create account attributed to video channel ' + actor.url)
  79         throw new Error(err)
  80       }
  81     }
  82
  83     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor)
  84     created = true
  85     accountPlaylistsUrl = result.playlists
  86   }
  87
  88   if (actor.Account) actor.Account.Actor = actor
  89   if (actor.VideoChannel) actor.VideoChannel.Actor = actor
  90
  91   const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
  92   if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')
  93
  94   if ((created === true || refreshed === true) && updateCollections === true) {
  95     const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' }
  96     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
  97   }
  98
  99   // We created a new account: fetch the playlists
 100   if (created === true && actor.Account && accountPlaylistsUrl) {
 101     const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' }
 102     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 103   }
 104
 105   return actorRefreshed
 106 }
 107
 108 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
 109   return new ActorModel({
 110     type,
 111     url,
 112     preferredUsername,
 113     uuid,
 114     publicKey: null,
 115     privateKey: null,
 116     followersCount: 0,
 117     followingCount: 0,
 118     inboxUrl: url + '/inbox',
 119     outboxUrl: url + '/outbox',
 120     sharedInboxUrl: WEBSERVER.URL + '/inbox',
 121     followersUrl: url + '/followers',
 122     followingUrl: url + '/following'
 123   })
 124 }
 125
 126 async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) {
 127   const followersCount = await fetchActorTotalItems(attributes.followers)
 128   const followingCount = await fetchActorTotalItems(attributes.following)
 129
 130   actorInstance.type = attributes.type
 131   actorInstance.preferredUsername = attributes.preferredUsername
 132   actorInstance.url = attributes.id
 133   actorInstance.publicKey = attributes.publicKey.publicKeyPem
 134   actorInstance.followersCount = followersCount
 135   actorInstance.followingCount = followingCount
 136   actorInstance.inboxUrl = attributes.inbox
 137   actorInstance.outboxUrl = attributes.outbox
 138   actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox
 139   actorInstance.followersUrl = attributes.followers
 140   actorInstance.followingUrl = attributes.following
 141 }
 142
 143 async function updateActorAvatarInstance (actor: ActorModel, info: { name: string, onDisk: boolean, fileUrl: string }, t: Transaction) {
 144   if (info.name !== undefined) {
 145     if (actor.avatarId) {
 146       try {
 147         await actor.Avatar.destroy({ transaction: t })
 148       } catch (err) {
 149         logger.error('Cannot remove old avatar of actor %s.', actor.url, { err })
 150       }
 151     }
 152
 153     const avatar = await AvatarModel.create({
 154       filename: info.name,
 155       onDisk: info.onDisk,
 156       fileUrl: info.fileUrl
 157     }, { transaction: t })
 158
 159     actor.avatarId = avatar.id
 160     actor.Avatar = avatar
 161   }
 162
 163   return actor
 164 }
 165
 166 async function fetchActorTotalItems (url: string) {
 167   const options = {
 168     uri: url,
 169     method: 'GET',
 170     json: true,
 171     activityPub: true
 172   }
 173
 174   try {
 175     const { body } = await doRequest(options)
 176     return body.totalItems ? body.totalItems : 0
 177   } catch (err) {
 178     logger.warn('Cannot fetch remote actor count %s.', url, { err })
 179     return 0
 180   }
 181 }
 182
 183 async function getAvatarInfoIfExists (actorJSON: ActivityPubActor) {
 184   if (
 185     actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
 186     isActivityPubUrlValid(actorJSON.icon.url)
 187   ) {
 188     const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType]
 189
 190     return {
 191       name: uuidv4() + extension,
 192       fileUrl: actorJSON.icon.url
 193     }
 194   }
 195
 196   return undefined
 197 }
 198
 199 async function addFetchOutboxJob (actor: Pick<ActorModel, 'id' | 'outboxUrl'>) {
 200   // Don't fetch ourselves
 201   const serverActor = await getServerActor()
 202   if (serverActor.id === actor.id) {
 203     logger.error('Cannot fetch our own outbox!')
 204     return undefined
 205   }
 206
 207   const payload = {
 208     uri: actor.outboxUrl,
 209     type: 'activity' as 'activity'
 210   }
 211
 212   return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
 213 }
 214
 215 async function refreshActorIfNeeded (
 216   actorArg: ActorModel,
 217   fetchedType: ActorFetchByUrlType
 218 ): Promise<{ actor: ActorModel, refreshed: boolean }> {
 219   if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false }
 220
 221   // We need more attributes
 222   const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
 223
 224   try {
 225     let actorUrl: string
 226     try {
 227       actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost())
 228     } catch (err) {
 229       logger.warn('Cannot get actor URL from webfinger, keeping the old one.', err)
 230       actorUrl = actor.url
 231     }
 232
 233     const { result, statusCode } = await fetchRemoteActor(actorUrl)
 234
 235     if (statusCode === 404) {
 236       logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
 237       actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy()
 238       return { actor: undefined, refreshed: false }
 239     }
 240
 241     if (result === undefined) {
 242       logger.warn('Cannot fetch remote actor in refresh actor.')
 243       return { actor, refreshed: false }
 244     }
 245
 246     return sequelizeTypescript.transaction(async t => {
 247       updateInstanceWithAnother(actor, result.actor)
 248
 249       if (result.avatar !== undefined) {
 250         const avatarInfo = {
 251           name: result.avatar.name,
 252           fileUrl: result.avatar.fileUrl,
 253           onDisk: false
 254         }
 255
 256         await updateActorAvatarInstance(actor, avatarInfo, t)
 257       }
 258
 259       // Force update
 260       actor.setDataValue('updatedAt', new Date())
 261       await actor.save({ transaction: t })
 262
 263       if (actor.Account) {
 264         actor.Account.name = result.name
 265         actor.Account.description = result.summary
 266
 267         await actor.Account.save({ transaction: t })
 268       } else if (actor.VideoChannel) {
 269         actor.VideoChannel.name = result.name
 270         actor.VideoChannel.description = result.summary
 271         actor.VideoChannel.support = result.support
 272
 273         await actor.VideoChannel.save({ transaction: t })
 274       }
 275
 276       return { refreshed: true, actor }
 277     })
 278   } catch (err) {
 279     logger.warn('Cannot refresh actor %s.', actor.url, { err })
 280     return { actor, refreshed: false }
 281   }
 282 }
 283
 284 export {
 285   getOrCreateActorAndServerAndModel,
 286   buildActorInstance,
 287   setAsyncActorKeys,
 288   fetchActorTotalItems,
 289   getAvatarInfoIfExists,
 290   updateActorInstance,
 291   refreshActorIfNeeded,
 292   updateActorAvatarInstance,
 293   addFetchOutboxJob
 294 }
 295
 296 // ---------------------------------------------------------------------------
 297
 298 function saveActorAndServerAndModelIfNotExist (
 299   result: FetchRemoteActorResult,
 300   ownerActor?: ActorModel,
 301   t?: Transaction
 302 ): Bluebird<ActorModel> | Promise<ActorModel> {
 303   let actor = result.actor
 304
 305   if (t !== undefined) return save(t)
 306
 307   return sequelizeTypescript.transaction(t => save(t))
 308
 309   async function save (t: Transaction) {
 310     const actorHost = url.parse(actor.url).host
 311
 312     const serverOptions = {
 313       where: {
 314         host: actorHost
 315       },
 316       defaults: {
 317         host: actorHost
 318       },
 319       transaction: t
 320     }
 321     const [ server ] = await ServerModel.findOrCreate(serverOptions)
 322
 323     // Save our new account in database
 324     actor.serverId = server.id
 325
 326     // Avatar?
 327     if (result.avatar) {
 328       const avatar = await AvatarModel.create({
 329         filename: result.avatar.name,
 330         fileUrl: result.avatar.fileUrl,
 331         onDisk: false
 332       }, { transaction: t })
 333
 334       actor.avatarId = avatar.id
 335     }
 336
 337     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
 338     // (which could be false in a retried query)
 339     const [ actorCreated ] = await ActorModel.findOrCreate({
 340       defaults: actor.toJSON(),
 341       where: {
 342         url: actor.url
 343       },
 344       transaction: t
 345     })
 346
 347     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
 348       actorCreated.Account = await saveAccount(actorCreated, result, t)
 349       actorCreated.Account.Actor = actorCreated
 350     } else if (actorCreated.type === 'Group') { // Video channel
 351       actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t)
 352       actorCreated.VideoChannel.Actor = actorCreated
 353       actorCreated.VideoChannel.Account = ownerActor.Account
 354     }
 355
 356     actorCreated.Server = server
 357
 358     return actorCreated
 359   }
 360 }
 361
 362 type FetchRemoteActorResult = {
 363   actor: ActorModel
 364   name: string
 365   summary: string
 366   support?: string
 367   playlists?: string
 368   avatar?: {
 369     name: string,
 370     fileUrl: string
 371   }
 372   attributedTo: ActivityPubAttributedTo[]
 373 }
 374 async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> {
 375   const options = {
 376     uri: actorUrl,
 377     method: 'GET',
 378     json: true,
 379     activityPub: true
 380   }
 381
 382   logger.info('Fetching remote actor %s.', actorUrl)
 383
 384   const requestResult = await doRequest<ActivityPubActor>(options)
 385   const actorJSON = requestResult.body
 386
 387   if (sanitizeAndCheckActorObject(actorJSON) === false) {
 388     logger.debug('Remote actor JSON is not valid.', { actorJSON })
 389     return { result: undefined, statusCode: requestResult.response.statusCode }
 390   }
 391
 392   if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
 393     logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
 394     return { result: undefined, statusCode: requestResult.response.statusCode }
 395   }
 396
 397   const followersCount = await fetchActorTotalItems(actorJSON.followers)
 398   const followingCount = await fetchActorTotalItems(actorJSON.following)
 399
 400   const actor = new ActorModel({
 401     type: actorJSON.type,
 402     preferredUsername: actorJSON.preferredUsername,
 403     url: actorJSON.id,
 404     publicKey: actorJSON.publicKey.publicKeyPem,
 405     privateKey: null,
 406     followersCount: followersCount,
 407     followingCount: followingCount,
 408     inboxUrl: actorJSON.inbox,
 409     outboxUrl: actorJSON.outbox,
 410     sharedInboxUrl: actorJSON.endpoints.sharedInbox,
 411     followersUrl: actorJSON.followers,
 412     followingUrl: actorJSON.following
 413   })
 414
 415   const avatarInfo = await getAvatarInfoIfExists(actorJSON)
 416
 417   const name = actorJSON.name || actorJSON.preferredUsername
 418   return {
 419     statusCode: requestResult.response.statusCode,
 420     result: {
 421       actor,
 422       name,
 423       avatar: avatarInfo,
 424       summary: actorJSON.summary,
 425       support: actorJSON.support,
 426       playlists: actorJSON.playlists,
 427       attributedTo: actorJSON.attributedTo
 428     }
 429   }
 430 }
 431
 432 async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
 433   const [ accountCreated ] = await AccountModel.findOrCreate({
 434     defaults: {
 435       name: result.name,
 436       description: result.summary,
 437       actorId: actor.id
 438     },
 439     where: {
 440       actorId: actor.id
 441     },
 442     transaction: t
 443   })
 444
 445   return accountCreated
 446 }
 447
 448 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
 449   const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
 450     defaults: {
 451       name: result.name,
 452       description: result.summary,
 453       support: result.support,
 454       actorId: actor.id,
 455       accountId: ownerActor.Account.id
 456     },
 457     where: {
 458       actorId: actor.id
 459     },
 460     transaction: t
 461   })
 462
 463   return videoChannelCreated
 464 }