aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers/requests.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/helpers/requests.ts')
-rw-r--r--server/helpers/requests.ts199
1 files changed, 152 insertions, 47 deletions
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index b556c392e..fd2a56f30 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,58 +1,141 @@
1import * as Bluebird from 'bluebird'
2import { createWriteStream, remove } from 'fs-extra' 1import { createWriteStream, remove } from 'fs-extra'
3import * as request from 'request' 2import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
3import { join } from 'path'
4import { CONFIG } from '../initializers/config'
4import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' 5import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
6import { pipelinePromise } from './core-utils'
5import { processImage } from './image-utils' 7import { processImage } from './image-utils'
6import { join } from 'path'
7import { logger } from './logger' 8import { logger } from './logger'
8import { CONFIG } from '../initializers/config'
9 9
10function doRequest <T> ( 10export interface PeerTubeRequestError extends Error {
11 requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, 11 statusCode?: number
12 bodyKBLimit = 1000 // 1MB 12 responseBody?: any
13): Bluebird<{ response: request.RequestResponse, body: T }> { 13}
14 if (!(requestOptions.headers)) requestOptions.headers = {}
15 requestOptions.headers['User-Agent'] = getUserAgent()
16 14
17 if (requestOptions.activityPub === true) { 15const httpSignature = require('http-signature')
18 requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER 16
17type PeerTubeRequestOptions = {
18 activityPub?: boolean
19 bodyKBLimit?: number // 1MB
20 httpSignature?: {
21 algorithm: string
22 authorizationHeaderName: string
23 keyId: string
24 key: string
25 headers: string[]
19 } 26 }
27 jsonResponse?: boolean
28} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
29
30const peertubeGot = got.extend({
31 headers: {
32 'user-agent': getUserAgent()
33 },
34
35 handlers: [
36 (options, next) => {
37 const promiseOrStream = next(options) as CancelableRequest<any>
38 const bodyKBLimit = options.context?.bodyKBLimit as number
39 if (!bodyKBLimit) throw new Error('No KB limit for this request')
40
41 const bodyLimit = bodyKBLimit * 1000
42
43 /* eslint-disable @typescript-eslint/no-floating-promises */
44 promiseOrStream.on('downloadProgress', progress => {
45 if (progress.transferred > bodyLimit && progress.percent !== 1) {
46 const message = `Exceeded the download limit of ${bodyLimit} B`
47 logger.warn(message)
48
49 // CancelableRequest
50 if (promiseOrStream.cancel) {
51 promiseOrStream.cancel()
52 return
53 }
54
55 // Stream
56 (promiseOrStream as any).destroy()
57 }
58 })
20 59
21 return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { 60 return promiseOrStream
22 request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) 61 }
23 .on('data', onRequestDataLengthCheck(bodyKBLimit)) 62 ],
24 }) 63
64 hooks: {
65 beforeRequest: [
66 options => {
67 const headers = options.headers || {}
68 headers['host'] = options.url.host
69 },
70
71 options => {
72 const httpSignatureOptions = options.context?.httpSignature
73
74 if (httpSignatureOptions) {
75 const method = options.method ?? 'GET'
76 const path = options.path ?? options.url.pathname
77
78 if (!method || !path) {
79 throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
80 }
81
82 httpSignature.signRequest({
83 getHeader: function (header) {
84 return options.headers[header]
85 },
86
87 setHeader: function (header, value) {
88 options.headers[header] = value
89 },
90
91 method,
92 path
93 }, httpSignatureOptions)
94 }
95 }
96 ]
97 }
98})
99
100function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
101 const gotOptions = buildGotOptions(options)
102
103 return peertubeGot(url, gotOptions)
104 .catch(err => { throw buildRequestError(err) })
25} 105}
26 106
27function doRequestAndSaveToFile ( 107function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
28 requestOptions: request.CoreOptions & request.UriOptions, 108 const gotOptions = buildGotOptions(options)
109
110 return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
111 .catch(err => { throw buildRequestError(err) })
112}
113
114async function doRequestAndSaveToFile (
115 url: string,
29 destPath: string, 116 destPath: string,
30 bodyKBLimit = 10000 // 10MB 117 options: PeerTubeRequestOptions = {}
31) { 118) {
32 if (!requestOptions.headers) requestOptions.headers = {} 119 const gotOptions = buildGotOptions(options)
33 requestOptions.headers['User-Agent'] = getUserAgent()
34
35 return new Bluebird<void>((res, rej) => {
36 const file = createWriteStream(destPath)
37 file.on('finish', () => res())
38 120
39 request(requestOptions) 121 const outFile = createWriteStream(destPath)
40 .on('data', onRequestDataLengthCheck(bodyKBLimit))
41 .on('error', err => {
42 file.close()
43 122
44 remove(destPath) 123 try {
45 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) 124 await pipelinePromise(
125 peertubeGot.stream(url, gotOptions),
126 outFile
127 )
128 } catch (err) {
129 remove(destPath)
130 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
46 131
47 return rej(err) 132 throw buildRequestError(err)
48 }) 133 }
49 .pipe(file)
50 })
51} 134}
52 135
53async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { 136async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
54 const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) 137 const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
55 await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) 138 await doRequestAndSaveToFile(url, tmpPath)
56 139
57 const destPath = join(destDir, destName) 140 const destPath = join(destDir, destName)
58 141
@@ -73,24 +156,46 @@ function getUserAgent () {
73 156
74export { 157export {
75 doRequest, 158 doRequest,
159 doJSONRequest,
76 doRequestAndSaveToFile, 160 doRequestAndSaveToFile,
77 downloadImage 161 downloadImage
78} 162}
79 163
80// --------------------------------------------------------------------------- 164// ---------------------------------------------------------------------------
81 165
82// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 166function buildGotOptions (options: PeerTubeRequestOptions) {
83function onRequestDataLengthCheck (bodyKBLimit: number) { 167 const { activityPub, bodyKBLimit = 1000 } = options
84 let bufferLength = 0
85 const bytesLimit = bodyKBLimit * 1000
86 168
87 return function (chunk) { 169 const context = { bodyKBLimit, httpSignature: options.httpSignature }
88 bufferLength += chunk.length
89 if (bufferLength > bytesLimit) {
90 this.abort()
91 170
92 const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) 171 let headers = options.headers || {}
93 this.emit('error', error) 172
94 } 173 if (!headers.date) {
174 headers = { ...headers, date: new Date().toUTCString() }
175 }
176
177 if (activityPub && !headers.accept) {
178 headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
95 } 179 }
180
181 return {
182 method: options.method,
183 json: options.json,
184 searchParams: options.searchParams,
185 headers,
186 context
187 }
188}
189
190function buildRequestError (error: RequestError) {
191 const newError: PeerTubeRequestError = new Error(error.message)
192 newError.name = error.name
193 newError.stack = error.stack
194
195 if (error.response) {
196 newError.responseBody = error.response.body
197 newError.statusCode = error.response.statusCode
198 }
199
200 return newError
96} 201}