From 2f19481147b12e2ae503ed3d1f28621c94447ac3 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 7 Mar 2022 11:48:53 +0100 Subject: Optimize markdown renderer --- server/helpers/markdown.ts | 54 ++++++++++++++++++++++++---------------- server/tests/helpers/markdown.ts | 6 +++++ 2 files changed, 38 insertions(+), 22 deletions(-) (limited to 'server') diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts index 25685ec6d..f9291bdad 100644 --- a/server/helpers/markdown.ts +++ b/server/helpers/markdown.ts @@ -7,8 +7,13 @@ const sanitizeHtml = require('sanitize-html') const markdownItEmoji = require('markdown-it-emoji/light') const MarkdownItClass = require('markdown-it') -const markdownItWithHTML = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) -const markdownItWithoutHTML = new MarkdownItClass('default', { linkify: true, breaks: true, html: false }) +const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) + .enable(TEXT_WITH_HTML_RULES) + .use(markdownItEmoji) + +const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) + .use(markdownItEmoji) + .use(plainTextPlugin) const toSafeHtml = (text: string) => { if (!text) return '' @@ -17,9 +22,7 @@ const toSafeHtml = (text: string) => { const textWithLineFeed = text.replace(//g, '\r\n') // Convert possible markdown (emojis, emphasis and lists) to html - const html = markdownItWithHTML.enable(TEXT_WITH_HTML_RULES) - .use(markdownItEmoji) - .render(textWithLineFeed) + const html = markdownItForSafeHtml.render(textWithLineFeed) // Convert to safe Html return sanitizeHtml(html, defaultSanitizeOptions) @@ -28,12 +31,10 @@ const toSafeHtml = (text: string) => { const mdToOneLinePlainText = (text: string) => { if (!text) return '' - markdownItWithoutHTML.use(markdownItEmoji) - .use(plainTextPlugin) - .render(text) + markdownItForPlainText.render(text) // Convert to safe Html - return sanitizeHtml(markdownItWithoutHTML.plainText, textOnlySanitizeOptions) + return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions) } // --------------------------------------------------------------------------- @@ -47,30 +48,39 @@ export { // Thanks: https://github.com/wavesheep/markdown-it-plain-text function plainTextPlugin (markdownIt: any) { - let lastSeparator = '' - function plainTextRule (state: any) { const text = scan(state.tokens) - markdownIt.plainText = text.replace(/\s+/g, ' ') + // markdownIt.plainText = text.replace(/\s+/g, ' ') + markdownIt.plainText = text } function scan (tokens: any[]) { + let lastSeparator = '' let text = '' - for (const token of tokens) { - if (token.children !== null) { - text += scan(token.children) - continue - } - + function buildSeparator (token: any) { if (token.type === 'list_item_close') { lastSeparator = ', ' - } else if (/[a-zA-Z]+_close/.test(token.type)) { + } + + if (token.tag === 'br' || token.type === 'paragraph_close') { lastSeparator = ' ' - } else if (token.content) { - text += lastSeparator - text += token.content + } + } + + for (const token of tokens) { + buildSeparator(token) + + if (token.type !== 'inline') continue + + for (const child of token.children) { + buildSeparator(child) + + if (!child.content) continue + + text += lastSeparator + child.content + lastSeparator = '' } } diff --git a/server/tests/helpers/markdown.ts b/server/tests/helpers/markdown.ts index 0488a1a05..8177477f6 100644 --- a/server/tests/helpers/markdown.ts +++ b/server/tests/helpers/markdown.ts @@ -30,5 +30,11 @@ describe('Markdown helpers', function () { expect(result).to.equal('Hello coucou') }) + + it('Should convert tags to plain text', function () { + const result = mdToOneLinePlainText(`#déconversion\n#newage\n#histoire`) + + expect(result).to.equal('#déconversion #newage #histoire') + }) }) }) -- cgit v1.2.3 From 41878d33971473be3f4255c335b25b1ba869428f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 7 Mar 2022 11:52:29 +0100 Subject: Remove unnecessary comment --- server/helpers/markdown.ts | 1 - 1 file changed, 1 deletion(-) (limited to 'server') diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts index f9291bdad..a20ac22d4 100644 --- a/server/helpers/markdown.ts +++ b/server/helpers/markdown.ts @@ -51,7 +51,6 @@ function plainTextPlugin (markdownIt: any) { function plainTextRule (state: any) { const text = scan(state.tokens) - // markdownIt.plainText = text.replace(/\s+/g, ' ') markdownIt.plainText = text } -- cgit v1.2.3 From 0c058f256a195b92f124be10109c95d1fbe93ad8 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 7 Mar 2022 17:16:54 +0100 Subject: Remove exif tags when processing images --- server/helpers/image-utils.ts | 11 ++++++++ server/tests/fixtures/exif.jpg | Bin 0 -> 10877 bytes server/tests/fixtures/exif.png | Bin 0 -> 21059 bytes server/tests/helpers/image.ts | 58 ++++++++++++++++++++++++++++++++++------- 4 files changed, 59 insertions(+), 10 deletions(-) create mode 100644 server/tests/fixtures/exif.jpg create mode 100644 server/tests/fixtures/exif.png (limited to 'server') diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index b174ae436..28d8fff4c 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -80,6 +80,8 @@ async function autoResize (options: { const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight() const destIsPortraitOrSquare = newSize.width <= newSize.height + removeExif(sourceImage) + if (sourceIsPortrait && !destIsPortraitOrSquare) { const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height) .color([ { apply: 'shade', params: [ 50 ] } ]) @@ -106,6 +108,7 @@ function skipProcessing (options: { const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options const { width, height } = newSize + if (hasExif(sourceImage)) return false if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false if (inputExt !== outputExt) return false @@ -116,3 +119,11 @@ function skipProcessing (options: { return imageBytes <= 15 * kB } + +function hasExif (image: Jimp) { + return !!(image.bitmap as any).exifBuffer +} + +function removeExif (image: Jimp) { + (image.bitmap as any).exifBuffer = null +} diff --git a/server/tests/fixtures/exif.jpg b/server/tests/fixtures/exif.jpg new file mode 100644 index 000000000..2997b38e9 Binary files /dev/null and b/server/tests/fixtures/exif.jpg differ diff --git a/server/tests/fixtures/exif.png b/server/tests/fixtures/exif.png new file mode 100644 index 000000000..a1a0113f8 Binary files /dev/null and b/server/tests/fixtures/exif.png differ diff --git a/server/tests/helpers/image.ts b/server/tests/helpers/image.ts index 64bd373cc..475ca8fb2 100644 --- a/server/tests/helpers/image.ts +++ b/server/tests/helpers/image.ts @@ -4,6 +4,7 @@ import 'mocha' import { expect } from 'chai' import { readFile, remove } from 'fs-extra' import { join } from 'path' +import { execPromise } from '@server/helpers/core-utils' import { buildAbsoluteFixturePath, root } from '@shared/core-utils' import { processImage } from '../../../server/helpers/image-utils' @@ -20,40 +21,77 @@ async function checkBuffers (path1: string, path2: string, equals: boolean) { } } +async function hasTitleExif (path: string) { + const result = JSON.parse(await execPromise(`exiftool -json ${path}`)) + + return result[0]?.Title === 'should be removed' +} + describe('Image helpers', function () { const imageDestDir = join(root(), 'test-images') - const imageDest = join(imageDestDir, 'test.jpg') + + const imageDestJPG = join(imageDestDir, 'test.jpg') + const imageDestPNG = join(imageDestDir, 'test.png') + const thumbnailSize = { width: 223, height: 122 } it('Should skip processing if the source image is okay', async function () { const input = buildAbsoluteFixturePath('thumbnail.jpg') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, true) + await checkBuffers(input, imageDestJPG, true) }) it('Should not skip processing if the source image does not have the appropriate extension', async function () { const input = buildAbsoluteFixturePath('thumbnail.png') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, false) + await checkBuffers(input, imageDestJPG, false) }) it('Should not skip processing if the source image does not have the appropriate size', async function () { const input = buildAbsoluteFixturePath('preview.jpg') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, false) + await checkBuffers(input, imageDestJPG, false) }) it('Should not skip processing if the source image does not have the appropriate size', async function () { const input = buildAbsoluteFixturePath('thumbnail-big.jpg') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) + + await checkBuffers(input, imageDestJPG, false) + }) + + it('Should strip exif for a jpg file that can not be copied', async function () { + const input = buildAbsoluteFixturePath('exif.jpg') + expect(await hasTitleExif(input)).to.be.true + + await processImage(input, imageDestJPG, { width: 100, height: 100 }, true) + await checkBuffers(input, imageDestJPG, false) + + expect(await hasTitleExif(imageDestJPG)).to.be.false + }) + + it('Should strip exif for a jpg file that could be copied', async function () { + const input = buildAbsoluteFixturePath('exif.jpg') + expect(await hasTitleExif(input)).to.be.true + + await processImage(input, imageDestJPG, thumbnailSize, true) + await checkBuffers(input, imageDestJPG, false) + + expect(await hasTitleExif(imageDestJPG)).to.be.false + }) + + it('Should strip exif for png', async function () { + const input = buildAbsoluteFixturePath('exif.png') + expect(await hasTitleExif(input)).to.be.true - await checkBuffers(input, imageDest, false) + await processImage(input, imageDestPNG, thumbnailSize, true) + expect(await hasTitleExif(imageDestPNG)).to.be.false }) after(async function () { - await remove(imageDest) + await remove(imageDestDir) }) }) -- cgit v1.2.3 From 4c6d99e5b69b29e19d53ede14414b76beef815e2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 8 Mar 2022 08:50:38 +0100 Subject: Fix channel tests --- server/tests/fixtures/banner-resized.jpg | Bin 88780 -> 59947 bytes server/tests/shared/checks.ts | 12 ++++++------ 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'server') diff --git a/server/tests/fixtures/banner-resized.jpg b/server/tests/fixtures/banner-resized.jpg index 13ea422cb..952732d61 100644 Binary files a/server/tests/fixtures/banner-resized.jpg and b/server/tests/fixtures/banner-resized.jpg differ diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index 9ecc84b5d..8c8260088 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -25,21 +25,21 @@ async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { expect(content.toString()).to.not.contain(str) } -async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { +async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { const res = await makeGetRequest({ url, - path: imagePath, + path: imageHTTPPath, expectedStatus: HttpStatusCode.OK_200 }) const body = res.body const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) - const minLength = body.length - ((30 * body.length) / 100) - const maxLength = body.length + ((30 * body.length) / 100) + const minLength = data.length - ((30 * data.length) / 100) + const maxLength = data.length + ((30 * data.length) / 100) - expect(data.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') - expect(data.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') + expect(body.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') + expect(body.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') } async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { -- cgit v1.2.3 From c47c3bcb0a9265d2a414ec96075052f8619d3f21 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 8 Mar 2022 09:18:43 +0100 Subject: Fix multiple servers tests --- server/tests/api/videos/multiple-servers.ts | 2 +- server/tests/shared/checks.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'server') diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index ecdd36613..854958f80 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -207,7 +207,7 @@ describe('Test multiple servers', function () { }, { resolution: 720, - size: 788000 + size: 750000 } ], thumbnailfile: 'thumbnail', diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index 8c8260088..dcc16d7ea 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -35,8 +35,8 @@ async function testImage (url: string, imageName: string, imageHTTPPath: string, const body = res.body const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) - const minLength = data.length - ((30 * data.length) / 100) - const maxLength = data.length + ((30 * data.length) / 100) + const minLength = data.length - ((40 * data.length) / 100) + const maxLength = data.length + ((40 * data.length) / 100) expect(body.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') expect(body.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') -- cgit v1.2.3 From f6f48301a9ad392759d714d2cff13f1564942a50 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 8 Mar 2022 09:42:22 +0100 Subject: More robust transcoding checker --- server/middlewares/validators/videos/video-transcoding.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/middlewares/validators/videos/video-transcoding.ts b/server/middlewares/validators/videos/video-transcoding.ts index 34f231d45..da6638f4d 100644 --- a/server/middlewares/validators/videos/video-transcoding.ts +++ b/server/middlewares/validators/videos/video-transcoding.ts @@ -37,7 +37,7 @@ const createTranscodingValidator = [ // Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state const info = await VideoJobInfoModel.load(video.id) - if (info && info.pendingTranscode !== 0) { + if (info && info.pendingTranscode > 0) { return res.fail({ status: HttpStatusCode.CONFLICT_409, message: 'This video is already being transcoded' -- cgit v1.2.3