diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/helpers/image-utils.ts | 25 | ||||
-rw-r--r-- | server/initializers/constants.ts | 4 | ||||
-rw-r--r-- | server/lib/local-actor.ts | 5 | ||||
-rw-r--r-- | server/lib/thumbnail.ts | 27 | ||||
-rw-r--r-- | server/lib/worker/parent-process.ts | 16 | ||||
-rw-r--r-- | server/lib/worker/workers/image-downloader.ts | 2 | ||||
-rw-r--r-- | server/lib/worker/workers/image-processor.ts | 7 | ||||
-rw-r--r-- | server/tests/helpers/image.ts | 14 |
8 files changed, 75 insertions, 25 deletions
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index ebb102a0d..bbd4692ef 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts | |||
@@ -12,12 +12,14 @@ function generateImageFilename (extension = '.jpg') { | |||
12 | return buildUUID() + extension | 12 | return buildUUID() + extension |
13 | } | 13 | } |
14 | 14 | ||
15 | async function processImage ( | 15 | async function processImage (options: { |
16 | path: string, | 16 | path: string |
17 | destination: string, | 17 | destination: string |
18 | newSize: { width: number, height: number }, | 18 | newSize: { width: number, height: number } |
19 | keepOriginal = false | 19 | keepOriginal?: boolean // default false |
20 | ) { | 20 | }) { |
21 | const { path, destination, newSize, keepOriginal = false } = options | ||
22 | |||
21 | const extension = getLowercaseExtension(path) | 23 | const extension = getLowercaseExtension(path) |
22 | 24 | ||
23 | if (path === destination) { | 25 | if (path === destination) { |
@@ -36,7 +38,14 @@ async function processImage ( | |||
36 | if (keepOriginal !== true) await remove(path) | 38 | if (keepOriginal !== true) await remove(path) |
37 | } | 39 | } |
38 | 40 | ||
39 | async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) { | 41 | async function generateImageFromVideoFile (options: { |
42 | fromPath: string | ||
43 | folder: string | ||
44 | imageName: string | ||
45 | size: { width: number, height: number } | ||
46 | }) { | ||
47 | const { fromPath, folder, imageName, size } = options | ||
48 | |||
40 | const pendingImageName = 'pending-' + imageName | 49 | const pendingImageName = 'pending-' + imageName |
41 | const pendingImagePath = join(folder, pendingImageName) | 50 | const pendingImagePath = join(folder, pendingImageName) |
42 | 51 | ||
@@ -44,7 +53,7 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima | |||
44 | await generateThumbnailFromVideo(fromPath, folder, imageName) | 53 | await generateThumbnailFromVideo(fromPath, folder, imageName) |
45 | 54 | ||
46 | const destination = join(folder, imageName) | 55 | const destination = join(folder, imageName) |
47 | await processImage(pendingImagePath, destination, size) | 56 | await processImage({ path: pendingImagePath, destination, newSize: size }) |
48 | } catch (err) { | 57 | } catch (err) { |
49 | logger.error('Cannot generate image from video %s.', fromPath, { err, ...lTags() }) | 58 | logger.error('Cannot generate image from video %s.', fromPath, { err, ...lTags() }) |
50 | 59 | ||
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 175935835..c6989c38b 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -748,6 +748,10 @@ const WORKER_THREADS = { | |||
748 | DOWNLOAD_IMAGE: { | 748 | DOWNLOAD_IMAGE: { |
749 | CONCURRENCY: 3, | 749 | CONCURRENCY: 3, |
750 | MAX_THREADS: 1 | 750 | MAX_THREADS: 1 |
751 | }, | ||
752 | PROCESS_IMAGE: { | ||
753 | CONCURRENCY: 1, | ||
754 | MAX_THREADS: 5 | ||
751 | } | 755 | } |
752 | } | 756 | } |
753 | 757 | ||
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts index e3b04c094..1d9be76e2 100644 --- a/server/lib/local-actor.ts +++ b/server/lib/local-actor.ts | |||
@@ -6,14 +6,13 @@ import { getLowercaseExtension } from '@shared/core-utils' | |||
6 | import { buildUUID } from '@shared/extra-utils' | 6 | import { buildUUID } from '@shared/extra-utils' |
7 | import { ActivityPubActorType, ActorImageType } from '@shared/models' | 7 | import { ActivityPubActorType, ActorImageType } from '@shared/models' |
8 | import { retryTransactionWrapper } from '../helpers/database-utils' | 8 | import { retryTransactionWrapper } from '../helpers/database-utils' |
9 | import { processImage } from '../helpers/image-utils' | ||
10 | import { CONFIG } from '../initializers/config' | 9 | import { CONFIG } from '../initializers/config' |
11 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants' | 10 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants' |
12 | import { sequelizeTypescript } from '../initializers/database' | 11 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' | 12 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' |
14 | import { deleteActorImages, updateActorImages } from './activitypub/actors' | 13 | import { deleteActorImages, updateActorImages } from './activitypub/actors' |
15 | import { sendUpdateActor } from './activitypub/send' | 14 | import { sendUpdateActor } from './activitypub/send' |
16 | import { downloadImageFromWorker } from './worker/parent-process' | 15 | import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' |
17 | 16 | ||
18 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { | 17 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { |
19 | return new ActorModel({ | 18 | return new ActorModel({ |
@@ -42,7 +41,7 @@ async function updateLocalActorImageFiles ( | |||
42 | 41 | ||
43 | const imageName = buildUUID() + extension | 42 | const imageName = buildUUID() + extension |
44 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) | 43 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) |
45 | await processImage(imagePhysicalFile.path, destination, imageSize, true) | 44 | await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) |
46 | 45 | ||
47 | return { | 46 | return { |
48 | imageName, | 47 | imageName, |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index f00c87623..02b867a91 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { ThumbnailType } from '@shared/models' | 2 | import { ThumbnailType } from '@shared/models' |
3 | import { generateImageFilename, generateImageFromVideoFile, processImage } from '../helpers/image-utils' | 3 | import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils' |
4 | import { CONFIG } from '../initializers/config' | 4 | import { CONFIG } from '../initializers/config' |
5 | import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' | 5 | import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' |
6 | import { ThumbnailModel } from '../models/video/thumbnail' | 6 | import { ThumbnailModel } from '../models/video/thumbnail' |
@@ -9,6 +9,7 @@ import { MThumbnail } from '../types/models/video/thumbnail' | |||
9 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | 9 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' |
10 | import { downloadImageFromWorker } from './local-actor' | 10 | import { downloadImageFromWorker } from './local-actor' |
11 | import { VideoPathManager } from './video-path-manager' | 11 | import { VideoPathManager } from './video-path-manager' |
12 | import { processImageFromWorker } from './worker/parent-process' | ||
12 | 13 | ||
13 | type ImageSize = { height?: number, width?: number } | 14 | type ImageSize = { height?: number, width?: number } |
14 | 15 | ||
@@ -23,7 +24,10 @@ function updatePlaylistMiniatureFromExisting (options: { | |||
23 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) | 24 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) |
24 | const type = ThumbnailType.MINIATURE | 25 | const type = ThumbnailType.MINIATURE |
25 | 26 | ||
26 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | 27 | const thumbnailCreator = () => { |
28 | return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) | ||
29 | } | ||
30 | |||
27 | return updateThumbnailFromFunction({ | 31 | return updateThumbnailFromFunction({ |
28 | thumbnailCreator, | 32 | thumbnailCreator, |
29 | filename, | 33 | filename, |
@@ -99,7 +103,10 @@ function updateVideoMiniatureFromExisting (options: { | |||
99 | const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options | 103 | const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options |
100 | 104 | ||
101 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | 105 | const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) |
102 | const thumbnailCreator = () => processImage(inputPath, outputPath, { width, height }, keepOriginal) | 106 | |
107 | const thumbnailCreator = () => { | ||
108 | return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) | ||
109 | } | ||
103 | 110 | ||
104 | return updateThumbnailFromFunction({ | 111 | return updateThumbnailFromFunction({ |
105 | thumbnailCreator, | 112 | thumbnailCreator, |
@@ -123,8 +130,18 @@ function generateVideoMiniature (options: { | |||
123 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) | 130 | const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) |
124 | 131 | ||
125 | const thumbnailCreator = videoFile.isAudio() | 132 | const thumbnailCreator = videoFile.isAudio() |
126 | ? () => processImage(ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, outputPath, { width, height }, true) | 133 | ? () => processImageFromWorker({ |
127 | : () => generateImageFromVideoFile(input, basePath, filename, { height, width }) | 134 | path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, |
135 | destination: outputPath, | ||
136 | newSize: { width, height }, | ||
137 | keepOriginal: true | ||
138 | }) | ||
139 | : () => generateImageFromVideoFile({ | ||
140 | fromPath: input, | ||
141 | folder: basePath, | ||
142 | imageName: filename, | ||
143 | size: { height, width } | ||
144 | }) | ||
128 | 145 | ||
129 | return updateThumbnailFromFunction({ | 146 | return updateThumbnailFromFunction({ |
130 | thumbnailCreator, | 147 | thumbnailCreator, |
diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts index 18dabd97f..188001677 100644 --- a/server/lib/worker/parent-process.ts +++ b/server/lib/worker/parent-process.ts | |||
@@ -2,6 +2,7 @@ import { join } from 'path' | |||
2 | import Piscina from 'piscina' | 2 | import Piscina from 'piscina' |
3 | import { WORKER_THREADS } from '@server/initializers/constants' | 3 | import { WORKER_THREADS } from '@server/initializers/constants' |
4 | import { downloadImage } from './workers/image-downloader' | 4 | import { downloadImage } from './workers/image-downloader' |
5 | import { processImage } from '@server/helpers/image-utils' | ||
5 | 6 | ||
6 | const downloadImagerWorker = new Piscina({ | 7 | const downloadImagerWorker = new Piscina({ |
7 | filename: join(__dirname, 'workers', 'image-downloader.js'), | 8 | filename: join(__dirname, 'workers', 'image-downloader.js'), |
@@ -13,6 +14,19 @@ function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): | |||
13 | return downloadImagerWorker.run(options) | 14 | return downloadImagerWorker.run(options) |
14 | } | 15 | } |
15 | 16 | ||
17 | // --------------------------------------------------------------------------- | ||
18 | |||
19 | const processImageWorker = new Piscina({ | ||
20 | filename: join(__dirname, 'workers', 'image-processor.js'), | ||
21 | concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY, | ||
22 | maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS | ||
23 | }) | ||
24 | |||
25 | function processImageFromWorker (options: Parameters<typeof processImage>[0]): Promise<ReturnType<typeof processImage>> { | ||
26 | return processImageWorker.run(options) | ||
27 | } | ||
28 | |||
16 | export { | 29 | export { |
17 | downloadImageFromWorker | 30 | downloadImageFromWorker, |
31 | processImageFromWorker | ||
18 | } | 32 | } |
diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts index 8d4a6b37e..4b32f723e 100644 --- a/server/lib/worker/workers/image-downloader.ts +++ b/server/lib/worker/workers/image-downloader.ts | |||
@@ -18,7 +18,7 @@ async function downloadImage (options: { | |||
18 | const destPath = join(destDir, destName) | 18 | const destPath = join(destDir, destName) |
19 | 19 | ||
20 | try { | 20 | try { |
21 | await processImage(tmpPath, destPath, size) | 21 | await processImage({ path: tmpPath, destination: destPath, newSize: size }) |
22 | } catch (err) { | 22 | } catch (err) { |
23 | await remove(tmpPath) | 23 | await remove(tmpPath) |
24 | 24 | ||
diff --git a/server/lib/worker/workers/image-processor.ts b/server/lib/worker/workers/image-processor.ts new file mode 100644 index 000000000..0ab41a5a0 --- /dev/null +++ b/server/lib/worker/workers/image-processor.ts | |||
@@ -0,0 +1,7 @@ | |||
1 | import { processImage } from '@server/helpers/image-utils' | ||
2 | |||
3 | module.exports = processImage | ||
4 | |||
5 | export { | ||
6 | processImage | ||
7 | } | ||
diff --git a/server/tests/helpers/image.ts b/server/tests/helpers/image.ts index 475ca8fb2..7c5da69b5 100644 --- a/server/tests/helpers/image.ts +++ b/server/tests/helpers/image.ts | |||
@@ -37,28 +37,28 @@ describe('Image helpers', function () { | |||
37 | 37 | ||
38 | it('Should skip processing if the source image is okay', async function () { | 38 | it('Should skip processing if the source image is okay', async function () { |
39 | const input = buildAbsoluteFixturePath('thumbnail.jpg') | 39 | const input = buildAbsoluteFixturePath('thumbnail.jpg') |
40 | await processImage(input, imageDestJPG, thumbnailSize, true) | 40 | await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) |
41 | 41 | ||
42 | await checkBuffers(input, imageDestJPG, true) | 42 | await checkBuffers(input, imageDestJPG, true) |
43 | }) | 43 | }) |
44 | 44 | ||
45 | it('Should not skip processing if the source image does not have the appropriate extension', async function () { | 45 | it('Should not skip processing if the source image does not have the appropriate extension', async function () { |
46 | const input = buildAbsoluteFixturePath('thumbnail.png') | 46 | const input = buildAbsoluteFixturePath('thumbnail.png') |
47 | await processImage(input, imageDestJPG, thumbnailSize, true) | 47 | await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) |
48 | 48 | ||
49 | await checkBuffers(input, imageDestJPG, false) | 49 | await checkBuffers(input, imageDestJPG, false) |
50 | }) | 50 | }) |
51 | 51 | ||
52 | it('Should not skip processing if the source image does not have the appropriate size', async function () { | 52 | it('Should not skip processing if the source image does not have the appropriate size', async function () { |
53 | const input = buildAbsoluteFixturePath('preview.jpg') | 53 | const input = buildAbsoluteFixturePath('preview.jpg') |
54 | await processImage(input, imageDestJPG, thumbnailSize, true) | 54 | await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) |
55 | 55 | ||
56 | await checkBuffers(input, imageDestJPG, false) | 56 | await checkBuffers(input, imageDestJPG, false) |
57 | }) | 57 | }) |
58 | 58 | ||
59 | it('Should not skip processing if the source image does not have the appropriate size', async function () { | 59 | it('Should not skip processing if the source image does not have the appropriate size', async function () { |
60 | const input = buildAbsoluteFixturePath('thumbnail-big.jpg') | 60 | const input = buildAbsoluteFixturePath('thumbnail-big.jpg') |
61 | await processImage(input, imageDestJPG, thumbnailSize, true) | 61 | await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) |
62 | 62 | ||
63 | await checkBuffers(input, imageDestJPG, false) | 63 | await checkBuffers(input, imageDestJPG, false) |
64 | }) | 64 | }) |
@@ -67,7 +67,7 @@ describe('Image helpers', function () { | |||
67 | const input = buildAbsoluteFixturePath('exif.jpg') | 67 | const input = buildAbsoluteFixturePath('exif.jpg') |
68 | expect(await hasTitleExif(input)).to.be.true | 68 | expect(await hasTitleExif(input)).to.be.true |
69 | 69 | ||
70 | await processImage(input, imageDestJPG, { width: 100, height: 100 }, true) | 70 | await processImage({ path: input, destination: imageDestJPG, newSize: { width: 100, height: 100 }, keepOriginal: true }) |
71 | await checkBuffers(input, imageDestJPG, false) | 71 | await checkBuffers(input, imageDestJPG, false) |
72 | 72 | ||
73 | expect(await hasTitleExif(imageDestJPG)).to.be.false | 73 | expect(await hasTitleExif(imageDestJPG)).to.be.false |
@@ -77,7 +77,7 @@ describe('Image helpers', function () { | |||
77 | const input = buildAbsoluteFixturePath('exif.jpg') | 77 | const input = buildAbsoluteFixturePath('exif.jpg') |
78 | expect(await hasTitleExif(input)).to.be.true | 78 | expect(await hasTitleExif(input)).to.be.true |
79 | 79 | ||
80 | await processImage(input, imageDestJPG, thumbnailSize, true) | 80 | await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) |
81 | await checkBuffers(input, imageDestJPG, false) | 81 | await checkBuffers(input, imageDestJPG, false) |
82 | 82 | ||
83 | expect(await hasTitleExif(imageDestJPG)).to.be.false | 83 | expect(await hasTitleExif(imageDestJPG)).to.be.false |
@@ -87,7 +87,7 @@ describe('Image helpers', function () { | |||
87 | const input = buildAbsoluteFixturePath('exif.png') | 87 | const input = buildAbsoluteFixturePath('exif.png') |
88 | expect(await hasTitleExif(input)).to.be.true | 88 | expect(await hasTitleExif(input)).to.be.true |
89 | 89 | ||
90 | await processImage(input, imageDestPNG, thumbnailSize, true) | 90 | await processImage({ path: input, destination: imageDestPNG, newSize: thumbnailSize, keepOriginal: true }) |
91 | expect(await hasTitleExif(imageDestPNG)).to.be.false | 91 | expect(await hasTitleExif(imageDestPNG)).to.be.false |
92 | }) | 92 | }) |
93 | 93 | ||