]>
Commit | Line | Data |
---|---|---|
1 | import { createWriteStream, remove } from 'fs-extra' | |
2 | import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got' | |
3 | import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' | |
4 | import { join } from 'path' | |
5 | import { CONFIG } from '../initializers/config' | |
6 | import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants' | |
7 | import { pipelinePromise } from './core-utils' | |
8 | import { processImage } from './image-utils' | |
9 | import { logger, loggerTagsFactory } from './logger' | |
10 | import { getProxy, isProxyEnabled } from './proxy' | |
11 | ||
12 | const lTags = loggerTagsFactory('request') | |
13 | ||
14 | const httpSignature = require('@peertube/http-signature') | |
15 | ||
16 | export interface PeerTubeRequestError extends Error { | |
17 | statusCode?: number | |
18 | responseBody?: any | |
19 | responseHeaders?: any | |
20 | } | |
21 | ||
22 | type PeerTubeRequestOptions = { | |
23 | activityPub?: boolean | |
24 | bodyKBLimit?: number // 1MB | |
25 | httpSignature?: { | |
26 | algorithm: string | |
27 | authorizationHeaderName: string | |
28 | keyId: string | |
29 | key: string | |
30 | headers: string[] | |
31 | } | |
32 | jsonResponse?: boolean | |
33 | } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'> | |
34 | ||
35 | const peertubeGot = got.extend({ | |
36 | ...getAgent(), | |
37 | ||
38 | headers: { | |
39 | 'user-agent': getUserAgent() | |
40 | }, | |
41 | ||
42 | handlers: [ | |
43 | (options, next) => { | |
44 | const promiseOrStream = next(options) as CancelableRequest<any> | |
45 | const bodyKBLimit = options.context?.bodyKBLimit as number | |
46 | if (!bodyKBLimit) throw new Error('No KB limit for this request') | |
47 | ||
48 | const bodyLimit = bodyKBLimit * 1000 | |
49 | ||
50 | /* eslint-disable @typescript-eslint/no-floating-promises */ | |
51 | promiseOrStream.on('downloadProgress', progress => { | |
52 | if (progress.transferred > bodyLimit && progress.percent !== 1) { | |
53 | const message = `Exceeded the download limit of ${bodyLimit} B` | |
54 | logger.warn(message, lTags()) | |
55 | ||
56 | // CancelableRequest | |
57 | if (promiseOrStream.cancel) { | |
58 | promiseOrStream.cancel() | |
59 | return | |
60 | } | |
61 | ||
62 | // Stream | |
63 | (promiseOrStream as any).destroy() | |
64 | } | |
65 | }) | |
66 | ||
67 | return promiseOrStream | |
68 | } | |
69 | ], | |
70 | ||
71 | hooks: { | |
72 | beforeRequest: [ | |
73 | options => { | |
74 | const headers = options.headers || {} | |
75 | headers['host'] = options.url.host | |
76 | }, | |
77 | ||
78 | options => { | |
79 | const httpSignatureOptions = options.context?.httpSignature | |
80 | ||
81 | if (httpSignatureOptions) { | |
82 | const method = options.method ?? 'GET' | |
83 | const path = options.path ?? options.url.pathname | |
84 | ||
85 | if (!method || !path) { | |
86 | throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) | |
87 | } | |
88 | ||
89 | httpSignature.signRequest({ | |
90 | getHeader: function (header) { | |
91 | return options.headers[header] | |
92 | }, | |
93 | ||
94 | setHeader: function (header, value) { | |
95 | options.headers[header] = value | |
96 | }, | |
97 | ||
98 | method, | |
99 | path | |
100 | }, httpSignatureOptions) | |
101 | } | |
102 | } | |
103 | ], | |
104 | ||
105 | beforeRetry: [ | |
106 | (_options: NormalizedOptions, error: RequestError, retryCount: number) => { | |
107 | logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) | |
108 | } | |
109 | ] | |
110 | } | |
111 | }) | |
112 | ||
113 | function doRequest (url: string, options: PeerTubeRequestOptions = {}) { | |
114 | const gotOptions = buildGotOptions(options) | |
115 | ||
116 | return peertubeGot(url, gotOptions) | |
117 | .catch(err => { throw buildRequestError(err) }) | |
118 | } | |
119 | ||
120 | function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) { | |
121 | const gotOptions = buildGotOptions(options) | |
122 | ||
123 | return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' }) | |
124 | .catch(err => { throw buildRequestError(err) }) | |
125 | } | |
126 | ||
127 | async function doRequestAndSaveToFile ( | |
128 | url: string, | |
129 | destPath: string, | |
130 | options: PeerTubeRequestOptions = {} | |
131 | ) { | |
132 | const gotOptions = buildGotOptions(options) | |
133 | ||
134 | const outFile = createWriteStream(destPath) | |
135 | ||
136 | try { | |
137 | await pipelinePromise( | |
138 | peertubeGot.stream(url, gotOptions), | |
139 | outFile | |
140 | ) | |
141 | } catch (err) { | |
142 | remove(destPath) | |
143 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) | |
144 | ||
145 | throw buildRequestError(err) | |
146 | } | |
147 | } | |
148 | ||
149 | async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { | |
150 | const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) | |
151 | await doRequestAndSaveToFile(url, tmpPath) | |
152 | ||
153 | const destPath = join(destDir, destName) | |
154 | ||
155 | try { | |
156 | await processImage(tmpPath, destPath, size) | |
157 | } catch (err) { | |
158 | await remove(tmpPath) | |
159 | ||
160 | throw err | |
161 | } | |
162 | } | |
163 | ||
164 | function getAgent () { | |
165 | if (!isProxyEnabled()) return {} | |
166 | ||
167 | const proxy = getProxy() | |
168 | ||
169 | logger.info('Using proxy %s.', proxy, lTags()) | |
170 | ||
171 | const proxyAgentOptions = { | |
172 | keepAlive: true, | |
173 | keepAliveMsecs: 1000, | |
174 | maxSockets: 256, | |
175 | maxFreeSockets: 256, | |
176 | scheduling: 'lifo' as 'lifo', | |
177 | proxy | |
178 | } | |
179 | ||
180 | return { | |
181 | agent: { | |
182 | http: new HttpProxyAgent(proxyAgentOptions), | |
183 | https: new HttpsProxyAgent(proxyAgentOptions) | |
184 | } | |
185 | } | |
186 | } | |
187 | ||
188 | function getUserAgent () { | |
189 | return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` | |
190 | } | |
191 | ||
192 | function isBinaryResponse (result: Response<any>) { | |
193 | return BINARY_CONTENT_TYPES.has(result.headers['content-type']) | |
194 | } | |
195 | ||
196 | async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) { | |
197 | if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) | |
198 | ||
199 | const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) }) | |
200 | ||
201 | if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1) | |
202 | ||
203 | return url | |
204 | } | |
205 | ||
206 | // --------------------------------------------------------------------------- | |
207 | ||
208 | export { | |
209 | doRequest, | |
210 | doJSONRequest, | |
211 | doRequestAndSaveToFile, | |
212 | isBinaryResponse, | |
213 | downloadImage, | |
214 | findLatestRedirection, | |
215 | peertubeGot | |
216 | } | |
217 | ||
218 | // --------------------------------------------------------------------------- | |
219 | ||
220 | function buildGotOptions (options: PeerTubeRequestOptions) { | |
221 | const { activityPub, bodyKBLimit = 1000 } = options | |
222 | ||
223 | const context = { bodyKBLimit, httpSignature: options.httpSignature } | |
224 | ||
225 | let headers = options.headers || {} | |
226 | ||
227 | if (!headers.date) { | |
228 | headers = { ...headers, date: new Date().toUTCString() } | |
229 | } | |
230 | ||
231 | if (activityPub && !headers.accept) { | |
232 | headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } | |
233 | } | |
234 | ||
235 | return { | |
236 | method: options.method, | |
237 | dnsCache: true, | |
238 | timeout: REQUEST_TIMEOUT, | |
239 | json: options.json, | |
240 | searchParams: options.searchParams, | |
241 | retry: 2, | |
242 | headers, | |
243 | context | |
244 | } | |
245 | } | |
246 | ||
247 | function buildRequestError (error: RequestError) { | |
248 | const newError: PeerTubeRequestError = new Error(error.message) | |
249 | newError.name = error.name | |
250 | newError.stack = error.stack | |
251 | ||
252 | if (error.response) { | |
253 | newError.responseBody = error.response.body | |
254 | newError.responseHeaders = error.response.headers | |
255 | newError.statusCode = error.response.statusCode | |
256 | } | |
257 | ||
258 | return newError | |
259 | } |