Fix federation with some actors

That don't have a shared inbox, or a URL

13 files changed, 78 insertions, 33 deletions

diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts
index d9df253aa..ca42106b8 100644
--- a/server/controllers/activitypub/inbox.ts
+++ b/server/controllers/activitypub/inbox.ts
@@ -6,7 +6,6 @@ import { processActivities } from '../../lib/activitypub/process/process'
 import { asyncMiddleware, checkSignature, localAccountValidator, localVideoChannelValidator, signatureValidator } from '../../middlewares'
 import { activityPubValidator } from '../../middlewares/validators/activitypub/activity'
 import { queue } from 'async'
-import { ActorModel } from '../../models/activitypub/actor'
 import { MActorDefault, MActorSignature } from '../../typings/models'
 
 const inboxRouter = express.Router()
diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts
index 4e9aabf0e..ec4dd6f96 100644
--- a/server/helpers/custom-validators/activitypub/actor.ts
+++ b/server/helpers/custom-validators/activitypub/actor.ts
@@ -6,7 +6,12 @@ import { isHostValid } from '../servers'
 import { peertubeTruncate } from '@server/helpers/core-utils'
 
 function isActorEndpointsObjectValid (endpointObject: any) {
-  return isActivityPubUrlValid(endpointObject.sharedInbox)
+  if (endpointObject && endpointObject.sharedInbox) {
+    return isActivityPubUrlValid(endpointObject.sharedInbox)
+  }
+
+  // Shared inbox is optional
+  return true
 }
 
 function isActorPublicKeyObjectValid (publicKeyObject: any) {
@@ -16,7 +21,7 @@ function isActorPublicKeyObjectValid (publicKeyObject: any) {
 }
 
 function isActorTypeValid (type: string) {
-  return type === 'Person' || type === 'Application' || type === 'Group'
+  return type === 'Person' || type === 'Application' || type === 'Group' || type === 'Service' || type === 'Organization'
 }
 
 function isActorPublicKeyValid (publicKey: string) {
@@ -81,9 +86,11 @@ function sanitizeAndCheckActorObject (object: any) {
 }
 
 function normalizeActor (actor: any) {
-  if (!actor || !actor.url) return
+  if (!actor) return
 
-  if (typeof actor.url !== 'string') {
+  if (!actor.url) {
+    actor.url = actor.id
+  } else if (typeof actor.url !== 'string') {
     actor.url = actor.url.href || actor.url.url
   }
 
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 10f95f5ab..190fd427a 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 440
+const LAST_MIGRATION_VERSION = 445
 
 // ---------------------------------------------------------------------------
 
@@ -459,7 +459,9 @@ const ACTIVITY_PUB = {
 const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
   GROUP: 'Group',
   PERSON: 'Person',
-  APPLICATION: 'Application'
+  APPLICATION: 'Application',
+  ORGANIZATION: 'Organization',
+  SERVICE: 'Service'
 }
 
 const HTTP_SIGNATURE = {
diff --git a/server/initializers/migrations/0445-shared-inbox-optional.ts b/server/initializers/migrations/0445-shared-inbox-optional.ts
new file mode 100644
index 000000000..dad2d6569
--- /dev/null
+++ b/server/initializers/migrations/0445-shared-inbox-optional.ts
@@ -0,0 +1,26 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize,
+  db: any
+}): Promise<void> {
+  {
+    const data = {
+      type: Sequelize.STRING,
+      allowNull: true
+    }
+
+    await utils.queryInterface.changeColumn('actor', 'sharedInboxUrl', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 13b73077e..cad9af5e0 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -163,9 +163,12 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ
   actorInstance.followingCount = followingCount
   actorInstance.inboxUrl = attributes.inbox
   actorInstance.outboxUrl = attributes.outbox
-  actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox
   actorInstance.followersUrl = attributes.followers
   actorInstance.followingUrl = attributes.following
+
+  if (attributes.endpoints && attributes.endpoints.sharedInbox) {
+    actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox
+  }
 }
 
 type AvatarInfo = { name: string, onDisk: boolean, fileUrl: string }
@@ -437,9 +440,12 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
     followingCount: followingCount,
     inboxUrl: actorJSON.inbox,
     outboxUrl: actorJSON.outbox,
-    sharedInboxUrl: actorJSON.endpoints.sharedInbox,
     followersUrl: actorJSON.followers,
-    followingUrl: actorJSON.following
+    followingUrl: actorJSON.following,
+
+    sharedInboxUrl: actorJSON.endpoints && actorJSON.endpoints.sharedInbox
+      ? actorJSON.endpoints.sharedInbox
+      : null,
   })
 
   const avatarInfo = await getAvatarInfoIfExists(actorJSON)
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 26ec3e948..edbc14a73 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -100,7 +100,7 @@ async function sendCreateVideoComment (comment: MCommentOwnerVideo, t: Transacti
   if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
+  t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.getSharedInbox()))
 }
 
 function buildCreateActivity (url: string, byActor: MActorLight, object: any, audience?: ActivityAudience): ActivityCreate {
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 4b1ff8dc5..a91756ff4 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -71,7 +71,7 @@ async function sendDeleteVideoComment (videoComment: MCommentOwnerVideoReply, t:
   if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
+  t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.getSharedInbox()))
 }
 
 async function sendDeleteVideoPlaylist (videoPlaylist: MVideoPlaylistFullSummary, t: Transaction) {
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts
index 5ae1614ab..da7638a7b 100644
--- a/server/lib/activitypub/send/send-flag.ts
+++ b/server/lib/activitypub/send/send-flag.ts
@@ -18,7 +18,7 @@ async function sendVideoAbuse (byActor: MActor, videoAbuse: MVideoAbuseVideo, vi
   const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
   const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
 
-  t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl))
+  t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.getSharedInbox()))
 }
 
 function buildFlagActivity (url: string, byActor: MActor, videoAbuse: MVideoAbuseVideo, audience: ActivityAudience): ActivityFlag {
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index 8129ab32a..77b723479 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -7,7 +7,7 @@ import { JobQueue } from '../../job-queue'
 import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
 import { getServerActor } from '../../../helpers/utils'
 import { afterCommitIfTransaction } from '../../../helpers/database-utils'
-import { MActorFollowerException, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight } from '../../../typings/models'
+import { MActorWithInboxes, MActor, MActorId, MActorLight, MVideo, MVideoAccountLight } from '../../../typings/models'
 
 async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
   byActor: MActorLight,
@@ -24,7 +24,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
     const activity = activityBuilder(audience)
 
     return afterCommitIfTransaction(transaction, () => {
-      return unicastTo(activity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+      return unicastTo(activity, byActor, video.VideoChannel.Account.Actor.getSharedInbox())
     })
   }
 
@@ -40,7 +40,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
 async function forwardVideoRelatedActivity (
   activity: Activity,
   t: Transaction,
-  followersException: MActorFollowerException[] = [],
+  followersException: MActorWithInboxes[] = [],
   video: MVideo
 ) {
   // Mastodon does not add our announces in audience, so we forward to them manually
@@ -53,7 +53,7 @@ async function forwardVideoRelatedActivity (
 async function forwardActivity (
   activity: Activity,
   t: Transaction,
-  followersException: MActorFollowerException[] = [],
+  followersException: MActorWithInboxes[] = [],
   additionalFollowerUrls: string[] = []
 ) {
   logger.info('Forwarding activity %s.', activity.id)
@@ -90,7 +90,7 @@ async function broadcastToFollowers (
   byActor: MActorId,
   toFollowersOf: MActorId[],
   t: Transaction,
-  actorsException: MActorFollowerException[] = []
+  actorsException: MActorWithInboxes[] = []
 ) {
   const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
 
@@ -102,7 +102,7 @@ async function broadcastToActors (
   byActor: MActorId,
   toActors: MActor[],
   t?: Transaction,
-  actorsException: MActorFollowerException[] = []
+  actorsException: MActorWithInboxes[] = []
 ) {
   const uris = await computeUris(toActors, actorsException)
   return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
@@ -147,7 +147,7 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function computeFollowerUris (toFollowersOf: MActorId[], actorsException: MActorFollowerException[], t: Transaction) {
+async function computeFollowerUris (toFollowersOf: MActorId[], actorsException: MActorWithInboxes[], t: Transaction) {
   const toActorFollowerIds = toFollowersOf.map(a => a.id)
 
   const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t)
@@ -156,11 +156,11 @@ async function computeFollowerUris (toFollowersOf: MActorId[], actorsException:
   return result.data.filter(sharedInbox => sharedInboxesException.indexOf(sharedInbox) === -1)
 }
 
-async function computeUris (toActors: MActor[], actorsException: MActorFollowerException[] = []) {
+async function computeUris (toActors: MActor[], actorsException: MActorWithInboxes[] = []) {
   const serverActor = await getServerActor()
   const targetUrls = toActors
     .filter(a => a.id !== serverActor.id) // Don't send to ourselves
-    .map(a => a.sharedInboxUrl || a.inboxUrl)
+    .map(a => a.getSharedInbox())
 
   const toActorSharedInboxesSet = new Set(targetUrls)
 
@@ -169,10 +169,10 @@ async function computeUris (toActors: MActor[], actorsException: MActorFollowerE
               .filter(sharedInbox => sharedInboxesException.indexOf(sharedInbox) === -1)
 }
 
-async function buildSharedInboxesException (actorsException: MActorFollowerException[]) {
+async function buildSharedInboxesException (actorsException: MActorWithInboxes[]) {
   const serverActor = await getServerActor()
 
   return actorsException
-    .map(f => f.sharedInboxUrl || f.inboxUrl)
+    .map(f => f.getSharedInbox())
     .concat([ serverActor.sharedInboxUrl ])
 }
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index 8498692f0..fb3c4ef9d 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -574,8 +574,8 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     }
 
     const selections: string[] = []
-    if (distinct === true) selections.push('DISTINCT("Follows"."' + columnUrl + '") AS "url"')
-    else selections.push('"Follows"."' + columnUrl + '" AS "url"')
+    if (distinct === true) selections.push('DISTINCT("Follows"."' + columnUrl + '") AS "selectionUrl"')
+    else selections.push('"Follows"."' + columnUrl + '" AS "selectionUrl"')
 
     selections.push('COUNT(*) AS "total"')
 
@@ -585,7 +585,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
       let query = 'SELECT ' + selection + ' FROM "actor" ' +
         'INNER JOIN "actorFollow" ON "actorFollow"."' + firstJoin + '" = "actor"."id" ' +
         'INNER JOIN "actor" AS "Follows" ON "actorFollow"."' + secondJoin + '" = "Follows"."id" ' +
-        'WHERE "actor"."id" = ANY ($actorIds) AND "actorFollow"."state" = \'accepted\' '
+        'WHERE "actor"."id" = ANY ($actorIds) AND "actorFollow"."state" = \'accepted\' AND "selectionUrl" IS NOT NULL '
 
       if (count !== undefined) query += 'LIMIT ' + count
       if (start !== undefined) query += ' OFFSET ' + start
@@ -599,7 +599,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     }
 
     const [ followers, [ dataTotal ] ] = await Promise.all(tasks)
-    const urls: string[] = followers.map(f => f.url)
+    const urls: string[] = followers.map(f => f.selectionUrl)
 
     return {
       data: urls,
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 535ebd792..42a24b583 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -44,7 +44,8 @@ import {
   MActorFull,
   MActorHost,
   MActorServer,
-  MActorSummaryFormattable
+  MActorSummaryFormattable,
+  MActorWithInboxes
 } from '../../typings/models'
 import * as Bluebird from 'bluebird'
 
@@ -179,8 +180,8 @@ export class ActorModel extends Model<ActorModel> {
   @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max))
   outboxUrl: string
 
-  @AllowNull(false)
-  @Is('ActorSharedInboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'shared inbox url'))
+  @AllowNull(true)
+  @Is('ActorSharedInboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'shared inbox url', true))
   @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max))
   sharedInboxUrl: string
 
@@ -402,6 +403,10 @@ export class ActorModel extends Model<ActorModel> {
     })
   }
 
+  getSharedInbox (this: MActorWithInboxes) {
+    return this.sharedInboxUrl || this.inboxUrl
+  }
+
   toFormattedSummaryJSON (this: MActorSummaryFormattable) {
     let avatar: Avatar = null
     if (this.Avatar) {
diff --git a/server/typings/models/account/actor.ts b/server/typings/models/account/actor.ts
index bcacb8351..ee4ece755 100644
--- a/server/typings/models/account/actor.ts
+++ b/server/typings/models/account/actor.ts
@@ -19,7 +19,7 @@ export type MActorUsername = Pick<MActor, 'preferredUsername'>
 
 export type MActorFollowersUrl = Pick<MActor, 'followersUrl'>
 export type MActorAudience = MActorUrl & MActorFollowersUrl
-export type MActorFollowerException = Pick<ActorModel, 'sharedInboxUrl' | 'inboxUrl'>
+export type MActorWithInboxes = Pick<ActorModel, 'sharedInboxUrl' | 'inboxUrl' | 'getSharedInbox'>
 export type MActorSignature = MActorAccountChannelId
 
 export type MActorLight = Omit<MActor, 'privateKey' | 'privateKey'>
diff --git a/shared/models/activitypub/activitypub-actor.ts b/shared/models/activitypub/activitypub-actor.ts
index 53ec579bc..b8a2dc925 100644
--- a/shared/models/activitypub/activitypub-actor.ts
+++ b/shared/models/activitypub/activitypub-actor.ts
@@ -1,6 +1,6 @@
 import { ActivityPubAttributedTo } from './objects/common-objects'
 
-export type ActivityPubActorType = 'Person' | 'Application' | 'Group'
+export type ActivityPubActorType = 'Person' | 'Application' | 'Group' | 'Service' | 'Organization'
 
 export interface ActivityPubActor {
   '@context': any[]