aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/helpers/requests.ts41
-rw-r--r--server/lib/hls.ts3
-rw-r--r--server/tests/helpers/index.ts1
-rw-r--r--server/tests/helpers/request.ts48
4 files changed, 88 insertions, 5 deletions
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 5c6dc5e19..3762e4d3c 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,12 +1,14 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { createWriteStream } from 'fs-extra' 2import { createWriteStream, remove } from 'fs-extra'
3import * as request from 'request' 3import * as request from 'request'
4import { ACTIVITY_PUB, CONFIG } from '../initializers' 4import { ACTIVITY_PUB, CONFIG } from '../initializers'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
6import { join } from 'path' 6import { join } from 'path'
7import { logger } from './logger'
7 8
8function doRequest <T> ( 9function doRequest <T> (
9 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean } 10 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean },
11 bodyKBLimit = 1000 // 1MB
10): Bluebird<{ response: request.RequestResponse, body: T }> { 12): Bluebird<{ response: request.RequestResponse, body: T }> {
11 if (requestOptions.activityPub === true) { 13 if (requestOptions.activityPub === true) {
12 if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} 14 if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {}
@@ -15,16 +17,29 @@ function doRequest <T> (
15 17
16 return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { 18 return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => {
17 request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) 19 request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body }))
20 .on('data', onRequestDataLengthCheck(bodyKBLimit))
18 }) 21 })
19} 22}
20 23
21function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.UriOptions, destPath: string) { 24function doRequestAndSaveToFile (
25 requestOptions: request.CoreOptions & request.UriOptions,
26 destPath: string,
27 bodyKBLimit = 10000 // 10MB
28) {
22 return new Bluebird<void>((res, rej) => { 29 return new Bluebird<void>((res, rej) => {
23 const file = createWriteStream(destPath) 30 const file = createWriteStream(destPath)
24 file.on('finish', () => res()) 31 file.on('finish', () => res())
25 32
26 request(requestOptions) 33 request(requestOptions)
27 .on('error', err => rej(err)) 34 .on('data', onRequestDataLengthCheck(bodyKBLimit))
35 .on('error', err => {
36 file.close()
37
38 remove(destPath)
39 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
40
41 return rej(err)
42 })
28 .pipe(file) 43 .pipe(file)
29 }) 44 })
30} 45}
@@ -44,3 +59,21 @@ export {
44 doRequestAndSaveToFile, 59 doRequestAndSaveToFile,
45 downloadImage 60 downloadImage
46} 61}
62
63// ---------------------------------------------------------------------------
64
65// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3
66function onRequestDataLengthCheck (bodyKBLimit: number) {
67 let bufferLength = 0
68 const bytesLimit = bodyKBLimit * 1000
69
70 return function (chunk) {
71 bufferLength += chunk.length
72 if (bufferLength > bytesLimit) {
73 this.abort()
74
75 const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`)
76 this.emit('error', error)
77 }
78 }
79}
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 3575981f4..16a805ac2 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -116,7 +116,8 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
116 for (const fileUrl of fileUrls) { 116 for (const fileUrl of fileUrls) {
117 const destPath = join(tmpDirectory, basename(fileUrl)) 117 const destPath = join(tmpDirectory, basename(fileUrl))
118 118
119 await doRequestAndSaveToFile({ uri: fileUrl }, destPath) 119 const bodyKBLimit = 10 * 1000 * 1000 // 10GB
120 await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit)
120 } 121 }
121 122
122 clearTimeout(timer) 123 clearTimeout(timer)
diff --git a/server/tests/helpers/index.ts b/server/tests/helpers/index.ts
index 551208245..03b971770 100644
--- a/server/tests/helpers/index.ts
+++ b/server/tests/helpers/index.ts
@@ -1,2 +1,3 @@
1import './core-utils' 1import './core-utils'
2import './comment-model' 2import './comment-model'
3import './request'
diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts
new file mode 100644
index 000000000..95a74fdfa
--- /dev/null
+++ b/server/tests/helpers/request.ts
@@ -0,0 +1,48 @@
1/* tslint:disable:no-unused-expression */
2
3import 'mocha'
4import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
5import { get4KFileUrl, root, wait } from '../../../shared/utils'
6import { join } from 'path'
7import { pathExists, remove } from 'fs-extra'
8import { expect } from 'chai'
9
10describe('Request helpers', function () {
11 const destPath1 = join(root(), 'test-output-1.txt')
12 const destPath2 = join(root(), 'test-output-2.txt')
13
14 it('Should throw an error when the bytes limit is exceeded for request', async function () {
15 try {
16 await doRequest({ uri: get4KFileUrl() }, 3)
17 } catch {
18 return
19 }
20
21 throw new Error('No error thrown by do request')
22 })
23
24 it('Should throw an error when the bytes limit is exceeded for request and save file', async function () {
25 try {
26 await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath1, 3)
27 } catch {
28
29 await wait(500)
30 expect(await pathExists(destPath1)).to.be.false
31 return
32 }
33
34 throw new Error('No error thrown by do request and save to file')
35 })
36
37 it('Should succeed if the file is below the limit', async function () {
38 await doRequest({ uri: get4KFileUrl() }, 5)
39 await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath2, 5)
40
41 expect(await pathExists(destPath2)).to.be.true
42 })
43
44 after(async function () {
45 await remove(destPath1)
46 await remove(destPath2)
47 })
48})