aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--.github/actions/reusable-prepare-peertube-run/action.yml2
-rw-r--r--CHANGELOG.md18
-rw-r--r--client/package.json2
-rw-r--r--package.json2
-rw-r--r--server/helpers/image-utils.ts11
-rw-r--r--server/helpers/markdown.ts53
-rw-r--r--server/middlewares/validators/videos/video-transcoding.ts2
-rw-r--r--server/tests/fixtures/banner-resized.jpgbin88780 -> 59947 bytes
-rw-r--r--server/tests/fixtures/exif.jpgbin0 -> 10877 bytes
-rw-r--r--server/tests/fixtures/exif.pngbin0 -> 21059 bytes
-rw-r--r--server/tests/helpers/image.ts58
-rw-r--r--server/tests/helpers/markdown.ts6
-rw-r--r--server/tests/shared/checks.ts12
-rw-r--r--support/doc/development/tests.md8
-rw-r--r--support/docker/production/Dockerfile.bullseye2
15 files changed, 132 insertions, 44 deletions
diff --git a/.github/actions/reusable-prepare-peertube-run/action.yml b/.github/actions/reusable-prepare-peertube-run/action.yml
index 1a6cd2cfd..aa5b897c9 100644
--- a/.github/actions/reusable-prepare-peertube-run/action.yml
+++ b/.github/actions/reusable-prepare-peertube-run/action.yml
@@ -8,7 +8,7 @@ runs:
8 - name: Setup system dependencies 8 - name: Setup system dependencies
9 shell: bash 9 shell: bash
10 run: | 10 run: |
11 sudo apt-get install postgresql-client-common redis-tools parallel 11 sudo apt-get install postgresql-client-common redis-tools parallel libimage-exiftool-perl
12 wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" 12 wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz"
13 tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz 13 tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz
14 mkdir -p $HOME/bin 14 mkdir -p $HOME/bin
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2dfd9fd1c..e5e69b3f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,23 @@
1# Changelog 1# Changelog
2 2
3## v4.1.1
4
5### Security
6
7 * Strip EXIF data when processing images
8
9### Docker
10
11 * Fix videos import by installing python 3
12 * Install `git` package (may be needed to install some plugins)
13
14### Bug fixes
15
16 * Fix error when updating a live
17 * Fix performance regression when rendering HTML and feeds
18 * Fix player stuck by HTTP request error
19
20
3## v4.1.0 21## v4.1.0
4 22
5### IMPORTANT NOTES 23### IMPORTANT NOTES
diff --git a/client/package.json b/client/package.json
index 26ca15210..690e3b982 100644
--- a/client/package.json
+++ b/client/package.json
@@ -1,6 +1,6 @@
1{ 1{
2 "name": "peertube-client", 2 "name": "peertube-client",
3 "version": "4.1.0", 3 "version": "4.1.1",
4 "private": true, 4 "private": true,
5 "license": "AGPL-3.0", 5 "license": "AGPL-3.0",
6 "author": { 6 "author": {
diff --git a/package.json b/package.json
index d96f9675c..fd94620dd 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
1{ 1{
2 "name": "peertube", 2 "name": "peertube",
3 "description": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.", 3 "description": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.",
4 "version": "4.1.0", 4 "version": "4.1.1",
5 "private": true, 5 "private": true,
6 "licence": "AGPL-3.0", 6 "licence": "AGPL-3.0",
7 "engines": { 7 "engines": {
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts
index 9d0c09051..7d6451db9 100644
--- a/server/helpers/image-utils.ts
+++ b/server/helpers/image-utils.ts
@@ -118,6 +118,8 @@ async function autoResize (options: {
118 const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight() 118 const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight()
119 const destIsPortraitOrSquare = newSize.width <= newSize.height 119 const destIsPortraitOrSquare = newSize.width <= newSize.height
120 120
121 removeExif(sourceImage)
122
121 if (sourceIsPortrait && !destIsPortraitOrSquare) { 123 if (sourceIsPortrait && !destIsPortraitOrSquare) {
122 const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height) 124 const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height)
123 .color([ { apply: 'shade', params: [ 50 ] } ]) 125 .color([ { apply: 'shade', params: [ 50 ] } ])
@@ -144,6 +146,7 @@ function skipProcessing (options: {
144 const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options 146 const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options
145 const { width, height } = newSize 147 const { width, height } = newSize
146 148
149 if (hasExif(sourceImage)) return false
147 if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false 150 if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false
148 if (inputExt !== outputExt) return false 151 if (inputExt !== outputExt) return false
149 152
@@ -154,3 +157,11 @@ function skipProcessing (options: {
154 157
155 return imageBytes <= 15 * kB 158 return imageBytes <= 15 * kB
156} 159}
160
161function hasExif (image: Jimp) {
162 return !!(image.bitmap as any).exifBuffer
163}
164
165function removeExif (image: Jimp) {
166 (image.bitmap as any).exifBuffer = null
167}
diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts
index 41c1186ec..a20ac22d4 100644
--- a/server/helpers/markdown.ts
+++ b/server/helpers/markdown.ts
@@ -7,8 +7,13 @@ const sanitizeHtml = require('sanitize-html')
7const markdownItEmoji = require('markdown-it-emoji/light') 7const markdownItEmoji = require('markdown-it-emoji/light')
8const MarkdownItClass = require('markdown-it') 8const MarkdownItClass = require('markdown-it')
9 9
10const markdownItWithHTML = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) 10const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true })
11const markdownItWithoutHTML = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) 11 .enable(TEXT_WITH_HTML_RULES)
12 .use(markdownItEmoji)
13
14const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false })
15 .use(markdownItEmoji)
16 .use(plainTextPlugin)
12 17
13const toSafeHtml = (text: string) => { 18const toSafeHtml = (text: string) => {
14 if (!text) return '' 19 if (!text) return ''
@@ -17,9 +22,7 @@ const toSafeHtml = (text: string) => {
17 const textWithLineFeed = text.replace(/<br.?\/?>/g, '\r\n') 22 const textWithLineFeed = text.replace(/<br.?\/?>/g, '\r\n')
18 23
19 // Convert possible markdown (emojis, emphasis and lists) to html 24 // Convert possible markdown (emojis, emphasis and lists) to html
20 const html = markdownItWithHTML.enable(TEXT_WITH_HTML_RULES) 25 const html = markdownItForSafeHtml.render(textWithLineFeed)
21 .use(markdownItEmoji)
22 .render(textWithLineFeed)
23 26
24 // Convert to safe Html 27 // Convert to safe Html
25 return sanitizeHtml(html, defaultSanitizeOptions) 28 return sanitizeHtml(html, defaultSanitizeOptions)
@@ -28,12 +31,10 @@ const toSafeHtml = (text: string) => {
28const mdToOneLinePlainText = (text: string) => { 31const mdToOneLinePlainText = (text: string) => {
29 if (!text) return '' 32 if (!text) return ''
30 33
31 markdownItWithoutHTML.use(markdownItEmoji) 34 markdownItForPlainText.render(text)
32 .use(plainTextPlugin)
33 .render(text)
34 35
35 // Convert to safe Html 36 // Convert to safe Html
36 return sanitizeHtml(markdownItWithoutHTML.plainText, textOnlySanitizeOptions) 37 return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions)
37} 38}
38 39
39// --------------------------------------------------------------------------- 40// ---------------------------------------------------------------------------
@@ -47,30 +48,38 @@ export {
47 48
48// Thanks: https://github.com/wavesheep/markdown-it-plain-text 49// Thanks: https://github.com/wavesheep/markdown-it-plain-text
49function plainTextPlugin (markdownIt: any) { 50function plainTextPlugin (markdownIt: any) {
50 let lastSeparator = ''
51
52 function plainTextRule (state: any) { 51 function plainTextRule (state: any) {
53 const text = scan(state.tokens) 52 const text = scan(state.tokens)
54 53
55 markdownIt.plainText = text.replace(/\s+/g, ' ') 54 markdownIt.plainText = text
56 } 55 }
57 56
58 function scan (tokens: any[]) { 57 function scan (tokens: any[]) {
58 let lastSeparator = ''
59 let text = '' 59 let text = ''
60 60
61 for (const token of tokens) { 61 function buildSeparator (token: any) {
62 if (token.children !== null) {
63 text += scan(token.children)
64 continue
65 }
66
67 if (token.type === 'list_item_close') { 62 if (token.type === 'list_item_close') {
68 lastSeparator = ', ' 63 lastSeparator = ', '
69 } else if (token.type.endsWith('_close')) { 64 }
65
66 if (token.tag === 'br' || token.type === 'paragraph_close') {
70 lastSeparator = ' ' 67 lastSeparator = ' '
71 } else if (token.content) { 68 }
72 text += lastSeparator 69 }
73 text += token.content 70
71 for (const token of tokens) {
72 buildSeparator(token)
73
74 if (token.type !== 'inline') continue
75
76 for (const child of token.children) {
77 buildSeparator(child)
78
79 if (!child.content) continue
80
81 text += lastSeparator + child.content
82 lastSeparator = ''
74 } 83 }
75 } 84 }
76 85
diff --git a/server/middlewares/validators/videos/video-transcoding.ts b/server/middlewares/validators/videos/video-transcoding.ts
index 34f231d45..da6638f4d 100644
--- a/server/middlewares/validators/videos/video-transcoding.ts
+++ b/server/middlewares/validators/videos/video-transcoding.ts
@@ -37,7 +37,7 @@ const createTranscodingValidator = [
37 37
38 // Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state 38 // Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state
39 const info = await VideoJobInfoModel.load(video.id) 39 const info = await VideoJobInfoModel.load(video.id)
40 if (info && info.pendingTranscode !== 0) { 40 if (info && info.pendingTranscode > 0) {
41 return res.fail({ 41 return res.fail({
42 status: HttpStatusCode.CONFLICT_409, 42 status: HttpStatusCode.CONFLICT_409,
43 message: 'This video is already being transcoded' 43 message: 'This video is already being transcoded'
diff --git a/server/tests/fixtures/banner-resized.jpg b/server/tests/fixtures/banner-resized.jpg
index 13ea422cb..952732d61 100644
--- a/server/tests/fixtures/banner-resized.jpg
+++ b/server/tests/fixtures/banner-resized.jpg
Binary files differ
diff --git a/server/tests/fixtures/exif.jpg b/server/tests/fixtures/exif.jpg
new file mode 100644
index 000000000..2997b38e9
--- /dev/null
+++ b/server/tests/fixtures/exif.jpg
Binary files differ
diff --git a/server/tests/fixtures/exif.png b/server/tests/fixtures/exif.png
new file mode 100644
index 000000000..a1a0113f8
--- /dev/null
+++ b/server/tests/fixtures/exif.png
Binary files differ
diff --git a/server/tests/helpers/image.ts b/server/tests/helpers/image.ts
index 64bd373cc..475ca8fb2 100644
--- a/server/tests/helpers/image.ts
+++ b/server/tests/helpers/image.ts
@@ -4,6 +4,7 @@ import 'mocha'
4import { expect } from 'chai' 4import { expect } from 'chai'
5import { readFile, remove } from 'fs-extra' 5import { readFile, remove } from 'fs-extra'
6import { join } from 'path' 6import { join } from 'path'
7import { execPromise } from '@server/helpers/core-utils'
7import { buildAbsoluteFixturePath, root } from '@shared/core-utils' 8import { buildAbsoluteFixturePath, root } from '@shared/core-utils'
8import { processImage } from '../../../server/helpers/image-utils' 9import { processImage } from '../../../server/helpers/image-utils'
9 10
@@ -20,40 +21,77 @@ async function checkBuffers (path1: string, path2: string, equals: boolean) {
20 } 21 }
21} 22}
22 23
24async function hasTitleExif (path: string) {
25 const result = JSON.parse(await execPromise(`exiftool -json ${path}`))
26
27 return result[0]?.Title === 'should be removed'
28}
29
23describe('Image helpers', function () { 30describe('Image helpers', function () {
24 const imageDestDir = join(root(), 'test-images') 31 const imageDestDir = join(root(), 'test-images')
25 const imageDest = join(imageDestDir, 'test.jpg') 32
33 const imageDestJPG = join(imageDestDir, 'test.jpg')
34 const imageDestPNG = join(imageDestDir, 'test.png')
35
26 const thumbnailSize = { width: 223, height: 122 } 36 const thumbnailSize = { width: 223, height: 122 }
27 37
28 it('Should skip processing if the source image is okay', async function () { 38 it('Should skip processing if the source image is okay', async function () {
29 const input = buildAbsoluteFixturePath('thumbnail.jpg') 39 const input = buildAbsoluteFixturePath('thumbnail.jpg')
30 await processImage(input, imageDest, thumbnailSize, true) 40 await processImage(input, imageDestJPG, thumbnailSize, true)
31 41
32 await checkBuffers(input, imageDest, true) 42 await checkBuffers(input, imageDestJPG, true)
33 }) 43 })
34 44
35 it('Should not skip processing if the source image does not have the appropriate extension', async function () { 45 it('Should not skip processing if the source image does not have the appropriate extension', async function () {
36 const input = buildAbsoluteFixturePath('thumbnail.png') 46 const input = buildAbsoluteFixturePath('thumbnail.png')
37 await processImage(input, imageDest, thumbnailSize, true) 47 await processImage(input, imageDestJPG, thumbnailSize, true)
38 48
39 await checkBuffers(input, imageDest, false) 49 await checkBuffers(input, imageDestJPG, false)
40 }) 50 })
41 51
42 it('Should not skip processing if the source image does not have the appropriate size', async function () { 52 it('Should not skip processing if the source image does not have the appropriate size', async function () {
43 const input = buildAbsoluteFixturePath('preview.jpg') 53 const input = buildAbsoluteFixturePath('preview.jpg')
44 await processImage(input, imageDest, thumbnailSize, true) 54 await processImage(input, imageDestJPG, thumbnailSize, true)
45 55
46 await checkBuffers(input, imageDest, false) 56 await checkBuffers(input, imageDestJPG, false)
47 }) 57 })
48 58
49 it('Should not skip processing if the source image does not have the appropriate size', async function () { 59 it('Should not skip processing if the source image does not have the appropriate size', async function () {
50 const input = buildAbsoluteFixturePath('thumbnail-big.jpg') 60 const input = buildAbsoluteFixturePath('thumbnail-big.jpg')
51 await processImage(input, imageDest, thumbnailSize, true) 61 await processImage(input, imageDestJPG, thumbnailSize, true)
62
63 await checkBuffers(input, imageDestJPG, false)
64 })
65
66 it('Should strip exif for a jpg file that can not be copied', async function () {
67 const input = buildAbsoluteFixturePath('exif.jpg')
68 expect(await hasTitleExif(input)).to.be.true
69
70 await processImage(input, imageDestJPG, { width: 100, height: 100 }, true)
71 await checkBuffers(input, imageDestJPG, false)
72
73 expect(await hasTitleExif(imageDestJPG)).to.be.false
74 })
75
76 it('Should strip exif for a jpg file that could be copied', async function () {
77 const input = buildAbsoluteFixturePath('exif.jpg')
78 expect(await hasTitleExif(input)).to.be.true
79
80 await processImage(input, imageDestJPG, thumbnailSize, true)
81 await checkBuffers(input, imageDestJPG, false)
82
83 expect(await hasTitleExif(imageDestJPG)).to.be.false
84 })
85
86 it('Should strip exif for png', async function () {
87 const input = buildAbsoluteFixturePath('exif.png')
88 expect(await hasTitleExif(input)).to.be.true
52 89
53 await checkBuffers(input, imageDest, false) 90 await processImage(input, imageDestPNG, thumbnailSize, true)
91 expect(await hasTitleExif(imageDestPNG)).to.be.false
54 }) 92 })
55 93
56 after(async function () { 94 after(async function () {
57 await remove(imageDest) 95 await remove(imageDestDir)
58 }) 96 })
59}) 97})
diff --git a/server/tests/helpers/markdown.ts b/server/tests/helpers/markdown.ts
index 0488a1a05..8177477f6 100644
--- a/server/tests/helpers/markdown.ts
+++ b/server/tests/helpers/markdown.ts
@@ -30,5 +30,11 @@ describe('Markdown helpers', function () {
30 30
31 expect(result).to.equal('Hello coucou') 31 expect(result).to.equal('Hello coucou')
32 }) 32 })
33
34 it('Should convert tags to plain text', function () {
35 const result = mdToOneLinePlainText(`#déconversion\n#newage\n#histoire`)
36
37 expect(result).to.equal('#déconversion #newage #histoire')
38 })
33 }) 39 })
34}) 40})
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts
index 9ecc84b5d..dcc16d7ea 100644
--- a/server/tests/shared/checks.ts
+++ b/server/tests/shared/checks.ts
@@ -25,21 +25,21 @@ async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
25 expect(content.toString()).to.not.contain(str) 25 expect(content.toString()).to.not.contain(str)
26} 26}
27 27
28async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { 28async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') {
29 const res = await makeGetRequest({ 29 const res = await makeGetRequest({
30 url, 30 url,
31 path: imagePath, 31 path: imageHTTPPath,
32 expectedStatus: HttpStatusCode.OK_200 32 expectedStatus: HttpStatusCode.OK_200
33 }) 33 })
34 34
35 const body = res.body 35 const body = res.body
36 36
37 const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) 37 const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension))
38 const minLength = body.length - ((30 * body.length) / 100) 38 const minLength = data.length - ((40 * data.length) / 100)
39 const maxLength = body.length + ((30 * body.length) / 100) 39 const maxLength = data.length + ((40 * data.length) / 100)
40 40
41 expect(data.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') 41 expect(body.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture')
42 expect(data.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') 42 expect(body.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture')
43} 43}
44 44
45async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { 45async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) {
diff --git a/support/doc/development/tests.md b/support/doc/development/tests.md
index 02fc41147..47602156c 100644
--- a/support/doc/development/tests.md
+++ b/support/doc/development/tests.md
@@ -31,6 +31,12 @@ $ sudo docker run -p 9444:9000 chocobozzz/s3-ninja
31$ sudo docker run -p 10389:10389 chocobozzz/docker-test-openldap 31$ sudo docker run -p 10389:10389 chocobozzz/docker-test-openldap
32``` 32```
33 33
34Ensure you also have these commands:
35
36```
37$ exiftool --help
38```
39
34### Test 40### Test
35 41
36To run all test suites: 42To run all test suites:
@@ -39,7 +45,7 @@ To run all test suites:
39$ npm run test # See scripts/test.sh to run a particular suite 45$ npm run test # See scripts/test.sh to run a particular suite
40``` 46```
41 47
42Most of tests can be runned using: 48Most of tests can be run using:
43 49
44```bash 50```bash
45TS_NODE_TRANSPILE_ONLY=true npm run mocha -- --timeout 30000 --exit -r ts-node/register -r tsconfig-paths/register --bail server/tests/api/videos/video-transcoder.ts 51TS_NODE_TRANSPILE_ONLY=true npm run mocha -- --timeout 30000 --exit -r ts-node/register -r tsconfig-paths/register --bail server/tests/api/videos/video-transcoder.ts
diff --git a/support/docker/production/Dockerfile.bullseye b/support/docker/production/Dockerfile.bullseye
index e55da3307..c57b878ee 100644
--- a/support/docker/production/Dockerfile.bullseye
+++ b/support/docker/production/Dockerfile.bullseye
@@ -2,7 +2,7 @@ FROM node:14-bullseye-slim
2 2
3# Install dependencies 3# Install dependencies
4RUN apt update \ 4RUN apt update \
5 && apt install -y --no-install-recommends openssl ffmpeg python3 ca-certificates gnupg gosu build-essential curl \ 5 && apt install -y --no-install-recommends openssl ffmpeg python3 ca-certificates gnupg gosu build-essential curl git \
6 && gosu nobody true \ 6 && gosu nobody true \
7 && rm /var/lib/apt/lists/* -fR 7 && rm /var/lib/apt/lists/* -fR
8 8