Introduce worker threads to process remote images

1 files changed, 15 insertions, 20 deletions

diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts
index 01046d017..e3b04c094 100644
--- a/server/lib/local-actor.ts
+++ b/server/lib/local-actor.ts
@@ -1,4 +1,3 @@
-import { queue } from 'async'
 import { remove } from 'fs-extra'
 import LRUCache from 'lru-cache'
 import { join } from 'path'
@@ -8,13 +7,13 @@ import { buildUUID } from '@shared/extra-utils'
 import { ActivityPubActorType, ActorImageType } from '@shared/models'
 import { retryTransactionWrapper } from '../helpers/database-utils'
 import { processImage } from '../helpers/image-utils'
-import { downloadImage } from '../helpers/requests'
 import { CONFIG } from '../initializers/config'
-import { ACTOR_IMAGES_SIZE, LRU_CACHE, QUEUE_CONCURRENCY, WEBSERVER } from '../initializers/constants'
+import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants'
 import { sequelizeTypescript } from '../initializers/database'
 import { MAccountDefault, MActor, MChannelDefault } from '../types/models'
 import { deleteActorImages, updateActorImages } from './activitypub/actors'
 import { sendUpdateActor } from './activitypub/send'
+import { downloadImageFromWorker } from './worker/parent-process'
 
 function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) {
   return new ActorModel({
@@ -87,27 +86,22 @@ async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MC
   })
 }
 
-type DownloadImageQueueTask = {
+// ---------------------------------------------------------------------------
+
+function downloadActorImageFromWorker (options: {
   fileUrl: string
   filename: string
   type: ActorImageType
   size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0]
-}
-
-const downloadImageQueue = queue<DownloadImageQueueTask, Error>((task, cb) => {
-  downloadImage(task.fileUrl, CONFIG.STORAGE.ACTOR_IMAGES, task.filename, task.size)
-    .then(() => cb())
-    .catch(err => cb(err))
-}, QUEUE_CONCURRENCY.ACTOR_PROCESS_IMAGE)
-
-function pushActorImageProcessInQueue (task: DownloadImageQueueTask) {
-  return new Promise<void>((res, rej) => {
-    downloadImageQueue.push(task, err => {
-      if (err) return rej(err)
+}) {
+  const downloaderOptions = {
+    url: options.fileUrl,
+    destDir: CONFIG.STORAGE.ACTOR_IMAGES,
+    destName: options.filename,
+    size: options.size
+  }
 
-      return res()
-    })
-  })
+  return downloadImageFromWorker(downloaderOptions)
 }
 
 // Unsafe so could returns paths that does not exist anymore
@@ -116,7 +110,8 @@ const actorImagePathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.
 export {
   actorImagePathUnsafeCache,
   updateLocalActorImageFiles,
+  downloadActorImageFromWorker,
   deleteLocalActorImageFile,
-  pushActorImageProcessInQueue,
+  downloadImageFromWorker,
   buildActorInstance
 }