]>
Commit | Line | Data |
---|---|---|
bfe2ef6b | 1 | import { createWriteStream, remove } from 'fs-extra' |
3455c265 | 2 | import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got' |
8729a870 | 3 | import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' |
db4b15f2 C |
4 | import { join } from 'path' |
5 | import { CONFIG } from '../initializers/config' | |
4c99953a | 6 | import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants' |
db4b15f2 | 7 | import { pipelinePromise } from './core-utils' |
58d515e3 | 8 | import { processImage } from './image-utils' |
ac036180 | 9 | import { logger, loggerTagsFactory } from './logger' |
8729a870 | 10 | import { getProxy, isProxyEnabled } from './proxy' |
11 | ||
ac036180 C |
12 | const lTags = loggerTagsFactory('request') |
13 | ||
5842a854 | 14 | const httpSignature = require('@peertube/http-signature') |
da854ddd | 15 | |
b5c36108 C |
16 | export interface PeerTubeRequestError extends Error { |
17 | statusCode?: number | |
18 | responseBody?: any | |
3455c265 | 19 | responseHeaders?: any |
b5c36108 C |
20 | } |
21 | ||
db4b15f2 | 22 | type PeerTubeRequestOptions = { |
4c99953a | 23 | timeout?: number |
db4b15f2 C |
24 | activityPub?: boolean |
25 | bodyKBLimit?: number // 1MB | |
26 | httpSignature?: { | |
27 | algorithm: string | |
28 | authorizationHeaderName: string | |
29 | keyId: string | |
30 | key: string | |
31 | headers: string[] | |
32 | } | |
33 | jsonResponse?: boolean | |
34 | } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'> | |
35 | ||
36 | const peertubeGot = got.extend({ | |
8729a870 | 37 | ...getAgent(), |
38 | ||
db4b15f2 C |
39 | headers: { |
40 | 'user-agent': getUserAgent() | |
41 | }, | |
42 | ||
43 | handlers: [ | |
44 | (options, next) => { | |
45 | const promiseOrStream = next(options) as CancelableRequest<any> | |
b329abc2 | 46 | const bodyKBLimit = options.context?.bodyKBLimit as number |
db4b15f2 C |
47 | if (!bodyKBLimit) throw new Error('No KB limit for this request') |
48 | ||
b329abc2 C |
49 | const bodyLimit = bodyKBLimit * 1000 |
50 | ||
db4b15f2 C |
51 | /* eslint-disable @typescript-eslint/no-floating-promises */ |
52 | promiseOrStream.on('downloadProgress', progress => { | |
b329abc2 C |
53 | if (progress.transferred > bodyLimit && progress.percent !== 1) { |
54 | const message = `Exceeded the download limit of ${bodyLimit} B` | |
ac036180 | 55 | logger.warn(message, lTags()) |
b329abc2 C |
56 | |
57 | // CancelableRequest | |
58 | if (promiseOrStream.cancel) { | |
59 | promiseOrStream.cancel() | |
60 | return | |
61 | } | |
62 | ||
63 | // Stream | |
64 | (promiseOrStream as any).destroy() | |
db4b15f2 C |
65 | } |
66 | }) | |
e0ce715a | 67 | |
db4b15f2 C |
68 | return promiseOrStream |
69 | } | |
70 | ], | |
71 | ||
72 | hooks: { | |
73 | beforeRequest: [ | |
74 | options => { | |
75 | const headers = options.headers || {} | |
76 | headers['host'] = options.url.host | |
77 | }, | |
78 | ||
79 | options => { | |
80 | const httpSignatureOptions = options.context?.httpSignature | |
81 | ||
82 | if (httpSignatureOptions) { | |
83 | const method = options.method ?? 'GET' | |
84 | const path = options.path ?? options.url.pathname | |
85 | ||
86 | if (!method || !path) { | |
87 | throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) | |
88 | } | |
89 | ||
90 | httpSignature.signRequest({ | |
91 | getHeader: function (header) { | |
92 | return options.headers[header] | |
93 | }, | |
94 | ||
95 | setHeader: function (header, value) { | |
96 | options.headers[header] = value | |
97 | }, | |
98 | ||
99 | method, | |
100 | path | |
101 | }, httpSignatureOptions) | |
102 | } | |
103 | } | |
3455c265 C |
104 | ], |
105 | ||
106 | beforeRetry: [ | |
107 | (_options: NormalizedOptions, error: RequestError, retryCount: number) => { | |
108 | logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) | |
109 | } | |
db4b15f2 | 110 | ] |
da854ddd | 111 | } |
db4b15f2 | 112 | }) |
e4f97bab | 113 | |
db4b15f2 C |
114 | function doRequest (url: string, options: PeerTubeRequestOptions = {}) { |
115 | const gotOptions = buildGotOptions(options) | |
116 | ||
117 | return peertubeGot(url, gotOptions) | |
118 | .catch(err => { throw buildRequestError(err) }) | |
119 | } | |
120 | ||
121 | function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) { | |
122 | const gotOptions = buildGotOptions(options) | |
123 | ||
124 | return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' }) | |
125 | .catch(err => { throw buildRequestError(err) }) | |
e4f97bab | 126 | } |
dac0a531 | 127 | |
db4b15f2 C |
128 | async function doRequestAndSaveToFile ( |
129 | url: string, | |
bfe2ef6b | 130 | destPath: string, |
db4b15f2 | 131 | options: PeerTubeRequestOptions = {} |
bfe2ef6b | 132 | ) { |
4c99953a | 133 | const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE }) |
02988fdc | 134 | |
db4b15f2 | 135 | const outFile = createWriteStream(destPath) |
bfe2ef6b | 136 | |
db4b15f2 C |
137 | try { |
138 | await pipelinePromise( | |
139 | peertubeGot.stream(url, gotOptions), | |
140 | outFile | |
141 | ) | |
142 | } catch (err) { | |
143 | remove(destPath) | |
ac036180 | 144 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) |
bfe2ef6b | 145 | |
db4b15f2 C |
146 | throw buildRequestError(err) |
147 | } | |
0d0e8dd0 C |
148 | } |
149 | ||
6040f87d C |
150 | async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { |
151 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) | |
db4b15f2 | 152 | await doRequestAndSaveToFile(url, tmpPath) |
58d515e3 | 153 | |
6040f87d | 154 | const destPath = join(destDir, destName) |
4ee7a4c9 C |
155 | |
156 | try { | |
2fb5b3a5 | 157 | await processImage(tmpPath, destPath, size) |
4ee7a4c9 C |
158 | } catch (err) { |
159 | await remove(tmpPath) | |
160 | ||
161 | throw err | |
162 | } | |
58d515e3 C |
163 | } |
164 | ||
8729a870 | 165 | function getAgent () { |
166 | if (!isProxyEnabled()) return {} | |
167 | ||
168 | const proxy = getProxy() | |
169 | ||
ac036180 | 170 | logger.info('Using proxy %s.', proxy, lTags()) |
8729a870 | 171 | |
172 | const proxyAgentOptions = { | |
173 | keepAlive: true, | |
174 | keepAliveMsecs: 1000, | |
175 | maxSockets: 256, | |
176 | maxFreeSockets: 256, | |
177 | scheduling: 'lifo' as 'lifo', | |
178 | proxy | |
179 | } | |
180 | ||
181 | return { | |
182 | agent: { | |
183 | http: new HttpProxyAgent(proxyAgentOptions), | |
184 | https: new HttpsProxyAgent(proxyAgentOptions) | |
185 | } | |
186 | } | |
187 | } | |
188 | ||
e0ce715a | 189 | function getUserAgent () { |
66170ca8 | 190 | return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` |
e0ce715a C |
191 | } |
192 | ||
62549e6c C |
193 | function isBinaryResponse (result: Response<any>) { |
194 | return BINARY_CONTENT_TYPES.has(result.headers['content-type']) | |
195 | } | |
196 | ||
dedcd583 C |
197 | async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) { |
198 | if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) | |
199 | ||
200 | const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) }) | |
201 | ||
202 | if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1) | |
203 | ||
204 | return url | |
205 | } | |
206 | ||
9f10b292 | 207 | // --------------------------------------------------------------------------- |
dac0a531 | 208 | |
65fcc311 | 209 | export { |
e4f97bab | 210 | doRequest, |
db4b15f2 | 211 | doJSONRequest, |
58d515e3 | 212 | doRequestAndSaveToFile, |
62549e6c | 213 | isBinaryResponse, |
b3fc41a1 | 214 | downloadImage, |
dedcd583 | 215 | findLatestRedirection, |
b3fc41a1 | 216 | peertubeGot |
65fcc311 | 217 | } |
bfe2ef6b C |
218 | |
219 | // --------------------------------------------------------------------------- | |
220 | ||
db4b15f2 C |
221 | function buildGotOptions (options: PeerTubeRequestOptions) { |
222 | const { activityPub, bodyKBLimit = 1000 } = options | |
bfe2ef6b | 223 | |
db4b15f2 | 224 | const context = { bodyKBLimit, httpSignature: options.httpSignature } |
bfe2ef6b | 225 | |
db4b15f2 C |
226 | let headers = options.headers || {} |
227 | ||
e7053b1d C |
228 | if (!headers.date) { |
229 | headers = { ...headers, date: new Date().toUTCString() } | |
230 | } | |
db4b15f2 | 231 | |
e7053b1d | 232 | if (activityPub && !headers.accept) { |
db4b15f2 | 233 | headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } |
bfe2ef6b | 234 | } |
db4b15f2 C |
235 | |
236 | return { | |
237 | method: options.method, | |
bbb3be68 | 238 | dnsCache: true, |
4c99953a | 239 | timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT, |
db4b15f2 C |
240 | json: options.json, |
241 | searchParams: options.searchParams, | |
ac036180 | 242 | retry: 2, |
db4b15f2 C |
243 | headers, |
244 | context | |
245 | } | |
246 | } | |
247 | ||
b5c36108 C |
248 | function buildRequestError (error: RequestError) { |
249 | const newError: PeerTubeRequestError = new Error(error.message) | |
db4b15f2 C |
250 | newError.name = error.name |
251 | newError.stack = error.stack | |
252 | ||
b5c36108 C |
253 | if (error.response) { |
254 | newError.responseBody = error.response.body | |
3455c265 | 255 | newError.responseHeaders = error.response.headers |
b5c36108 | 256 | newError.statusCode = error.response.statusCode |
db4b15f2 C |
257 | } |
258 | ||
b5c36108 | 259 | return newError |
bfe2ef6b | 260 | } |