Introduce worker threads to process remote images

2 files changed, 51 insertions, 0 deletions

diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts
new file mode 100644
index 000000000..18dabd97f
--- /dev/null
+++ b/server/lib/worker/parent-process.ts
@@ -0,0 +1,18 @@
+import { join } from 'path'
+import Piscina from 'piscina'
+import { WORKER_THREADS } from '@server/initializers/constants'
+import { downloadImage } from './workers/image-downloader'
+
+const downloadImagerWorker = new Piscina({
+  filename: join(__dirname, 'workers', 'image-downloader.js'),
+  concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,
+  maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS
+})
+
+function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> {
+  return downloadImagerWorker.run(options)
+}
+
+export {
+  downloadImageFromWorker
+}
diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts
new file mode 100644
index 000000000..8d4a6b37e
--- /dev/null
+++ b/server/lib/worker/workers/image-downloader.ts
@@ -0,0 +1,33 @@
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { processImage } from '@server/helpers/image-utils'
+import { doRequestAndSaveToFile } from '@server/helpers/requests'
+import { CONFIG } from '@server/initializers/config'
+
+async function downloadImage (options: {
+  url: string
+  destDir: string
+  destName: string
+  size: { width: number, height: number }
+}) {
+  const { url, destDir, destName, size } = options
+
+  const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
+  await doRequestAndSaveToFile(url, tmpPath)
+
+  const destPath = join(destDir, destName)
+
+  try {
+    await processImage(tmpPath, destPath, size)
+  } catch (err) {
+    await remove(tmpPath)
+
+    throw err
+  }
+}
+
+module.exports = downloadImage
+
+export {
+  downloadImage
+}