Fix max buffer reached in youtube import

[github/Chocobozzz/PeerTube.git] / server / lib / activitypub / actor.ts

diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index b3fb75421e7651004503b7b03ecc19478ba9390b..c3255d8ca1ac408e95b0b07b73626969abbda005 100644 (file)
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -12,7 +12,7 @@ import { logger } from '../../helpers/logger'
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
 import { getUrlFromWebfinger } from '../../helpers/webfinger'
-import { AVATAR_MIMETYPE_EXT, CONFIG, sequelizeTypescript } from '../../initializers'
+import { IMAGE_MIMETYPE_EXT, CONFIG, sequelizeTypescript } from '../../initializers'
 import { AccountModel } from '../../models/account/account'
 import { ActorModel } from '../../models/activitypub/actor'
 import { AvatarModel } from '../../models/avatar/avatar'
@@ -64,7 +64,11 @@ async function getOrCreateActorAndServerAndModel (actorUrl: string, recurseIfNee
     actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, options)
   }
 
-  return refreshActorIfNeeded(actor)
+  const options = {
+    arguments: [ actor ],
+    errorMessage: 'Cannot refresh actor if needed with many retries.'
+  }
+  return retryTransactionWrapper(refreshActorIfNeeded, options)
 }
 
 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) {
@@ -132,23 +136,21 @@ async function fetchActorTotalItems (url: string) {
     activityPub: true
   }
 
-  let requestResult
   try {
-    requestResult = await doRequest(options)
+    const { body } = await doRequest(options)
+    return body.totalItems ? body.totalItems : 0
   } catch (err) {
     logger.warn('Cannot fetch remote actor count %s.', url, err)
-    return undefined
+    return 0
   }
-
-  return requestResult.totalItems ? requestResult.totalItems : 0
 }
 
 async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
   if (
-    actorJSON.icon && actorJSON.icon.type === 'Image' && AVATAR_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
+    actorJSON.icon && actorJSON.icon.type === 'Image' && IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined &&
     isActivityPubUrlValid(actorJSON.icon.url)
   ) {
-    const extension = AVATAR_MIMETYPE_EXT[actorJSON.icon.mediaType]
+    const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
 
     const avatarName = uuidv4() + extension
     const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
@@ -214,7 +216,13 @@ function saveActorAndServerAndModelIfNotExist (
 
     // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists
     // (which could be false in a retried query)
-    const actorCreated = await ActorModel.create(actor.toJSON(), { transaction: t })
+    const [ actorCreated ] = await ActorModel.findOrCreate({
+      defaults: actor.toJSON(),
+      where: {
+        url: actor.url
+      },
+      transaction: t
+    })
 
     if (actorCreated.type === 'Person' || actorCreated.type === 'Application') {
       const account = await saveAccount(actorCreated, result, t)
@@ -248,7 +256,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<FetchRemoteActorResu
   logger.info('Fetching remote actor %s.', actorUrl)
 
   const requestResult = await doRequest(options)
-  const actorJSON: ActivityPubActor = requestResult.body
+  const actorJSON: ActivityPubActor = normalizeActor(requestResult.body)
 
   if (isActorObjectValid(actorJSON) === false) {
     logger.debug('Remote actor JSON is not valid.', { actorJSON: actorJSON })
@@ -286,57 +294,83 @@ async function fetchRemoteActor (actorUrl: string): Promise<FetchRemoteActorResu
   }
 }
 
-function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
-  const account = new AccountModel({
-    name: result.name,
-    actorId: actor.id
+async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) {
+  const [ accountCreated ] = await AccountModel.findOrCreate({
+    defaults: {
+      name: result.name,
+      actorId: actor.id
+    },
+    where: {
+      actorId: actor.id
+    },
+    transaction: t
   })
 
-  return account.save({ transaction: t })
+  return accountCreated
 }
 
 async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) {
-  const videoChannel = new VideoChannelModel({
-    name: result.name,
-    description: result.summary,
-    actorId: actor.id,
-    accountId: ownerActor.Account.id
+  const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({
+    defaults: {
+      name: result.name,
+      description: result.summary,
+      actorId: actor.id,
+      accountId: ownerActor.Account.id
+    },
+    where: {
+      actorId: actor.id
+    },
+    transaction: t
   })
 
-  return videoChannel.save({ transaction: t })
+  return videoChannelCreated
 }
 
 async function refreshActorIfNeeded (actor: ActorModel) {
   if (!actor.isOutdated()) return actor
 
-  const actorUrl = await getUrlFromWebfinger(actor.preferredUsername, actor.getHost())
-  const result = await fetchRemoteActor(actorUrl)
-  if (result === undefined) throw new Error('Cannot fetch remote actor in refresh actor.')
-
-  return sequelizeTypescript.transaction(async t => {
-    logger.info('coucou', result.actor.toJSON())
-    updateInstanceWithAnother(actor, result.actor)
-
-    if (result.avatarName !== undefined) {
-      await updateActorAvatarInstance(actor, result.avatarName, t)
+  try {
+    const actorUrl = await getUrlFromWebfinger(actor.preferredUsername, actor.getHost())
+    const result = await fetchRemoteActor(actorUrl)
+    if (result === undefined) {
+      logger.warn('Cannot fetch remote actor in refresh actor.')
+      return actor
     }
 
-    // Force update
-    actor.setDataValue('updatedAt', new Date())
-    await actor.save({ transaction: t })
+    return sequelizeTypescript.transaction(async t => {
+      updateInstanceWithAnother(actor, result.actor)
 
-    if (actor.Account) {
-      await actor.save({ transaction: t })
+      if (result.avatarName !== undefined) {
+        await updateActorAvatarInstance(actor, result.avatarName, t)
+      }
 
-      actor.Account.set('name', result.name)
-      await actor.Account.save({ transaction: t })
-    } else if (actor.VideoChannel) {
+      // Force update
+      actor.setDataValue('updatedAt', new Date())
       await actor.save({ transaction: t })
 
-      actor.VideoChannel.set('name', result.name)
-      await actor.VideoChannel.save({ transaction: t })
-    }
+      if (actor.Account) {
+        await actor.save({ transaction: t })
 
+        actor.Account.set('name', result.name)
+        await actor.Account.save({ transaction: t })
+      } else if (actor.VideoChannel) {
+        await actor.save({ transaction: t })
+
+        actor.VideoChannel.set('name', result.name)
+        await actor.VideoChannel.save({ transaction: t })
+      }
+
+      return actor
+    })
+  } catch (err) {
+    logger.warn('Cannot refresh actor.', err)
     return actor
-  })
+  }
+}
+
+function normalizeActor (actor: any) {
+  if (actor && actor.url && typeof actor.url === 'string') return actor
+
+  actor.url = actor.url.href || actor.url.url
+  return actor
 }