]> git.immae.eu Git - github/shaarli/Shaarli.git/blob - application/HttpUtils.php
LinkDB: update datastore method names
[github/shaarli/Shaarli.git] / application / HttpUtils.php
1 <?php
2 /**
3 * GET an HTTP URL to retrieve its content
4 * Uses the cURL library or a fallback method
5 *
6 * @param string $url URL to get (http://...)
7 * @param int $timeout network timeout (in seconds)
8 * @param int $maxBytes maximum downloaded bytes (default: 4 MiB)
9 *
10 * @return array HTTP response headers, downloaded content
11 *
12 * Output format:
13 * [0] = associative array containing HTTP response headers
14 * [1] = URL content (downloaded data)
15 *
16 * Example:
17 * list($headers, $data) = get_http_response('http://sebauvage.net/');
18 * if (strpos($headers[0], '200 OK') !== false) {
19 * echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
20 * } else {
21 * echo 'There was an error: '.htmlspecialchars($headers[0]);
22 * }
23 *
24 * @see https://secure.php.net/manual/en/ref.curl.php
25 * @see https://secure.php.net/manual/en/functions.anonymous.php
26 * @see https://secure.php.net/manual/en/function.preg-split.php
27 * @see https://secure.php.net/manual/en/function.explode.php
28 * @see http://stackoverflow.com/q/17641073
29 * @see http://stackoverflow.com/q/9183178
30 * @see http://stackoverflow.com/q/1462720
31 */
32 function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
33 {
34 $urlObj = new Url($url);
35 $cleanUrl = $urlObj->idnToAscii();
36
37 if (!filter_var($cleanUrl, FILTER_VALIDATE_URL) || !$urlObj->isHttp()) {
38 return array(array(0 => 'Invalid HTTP Url'), false);
39 }
40
41 $userAgent =
42 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
43 . ' Gecko/20100101 Firefox/45.0';
44 $acceptLanguage =
45 substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3';
46 $maxRedirs = 3;
47
48 if (!function_exists('curl_init')) {
49 return get_http_response_fallback(
50 $cleanUrl,
51 $timeout,
52 $maxBytes,
53 $userAgent,
54 $acceptLanguage,
55 $maxRedirs
56 );
57 }
58
59 $ch = curl_init($cleanUrl);
60 if ($ch === false) {
61 return array(array(0 => 'curl_init() error'), false);
62 }
63
64 // General cURL settings
65 curl_setopt($ch, CURLOPT_AUTOREFERER, true);
66 curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
67 curl_setopt($ch, CURLOPT_HEADER, true);
68 curl_setopt(
69 $ch,
70 CURLOPT_HTTPHEADER,
71 array('Accept-Language: ' . $acceptLanguage)
72 );
73 curl_setopt($ch, CURLOPT_MAXREDIRS, $maxRedirs);
74 curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
75 curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
76 curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
77
78 // Max download size management
79 curl_setopt($ch, CURLOPT_BUFFERSIZE, 1024);
80 curl_setopt($ch, CURLOPT_NOPROGRESS, false);
81 curl_setopt($ch, CURLOPT_PROGRESSFUNCTION,
82 function($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes)
83 {
84 if (version_compare(phpversion(), '5.5', '<')) {
85 // PHP version lower than 5.5
86 // Callback has 4 arguments
87 $downloaded = $arg1;
88 } else {
89 // Callback has 5 arguments
90 $downloaded = $arg2;
91 }
92 // Non-zero return stops downloading
93 return ($downloaded > $maxBytes) ? 1 : 0;
94 }
95 );
96
97 $response = curl_exec($ch);
98 $errorNo = curl_errno($ch);
99 $errorStr = curl_error($ch);
100 $headSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
101 curl_close($ch);
102
103 if ($response === false) {
104 if ($errorNo == CURLE_COULDNT_RESOLVE_HOST) {
105 /*
106 * Workaround to match fallback method behaviour
107 * Removing this would require updating
108 * GetHttpUrlTest::testGetInvalidRemoteUrl()
109 */
110 return array(false, false);
111 }
112 return array(array(0 => 'curl_exec() error: ' . $errorStr), false);
113 }
114
115 // Formatting output like the fallback method
116 $rawHeaders = substr($response, 0, $headSize);
117
118 // Keep only headers from latest redirection
119 $rawHeadersArrayRedirs = explode("\r\n\r\n", trim($rawHeaders));
120 $rawHeadersLastRedir = end($rawHeadersArrayRedirs);
121
122 $content = substr($response, $headSize);
123 $headers = array();
124 foreach (preg_split('~[\r\n]+~', $rawHeadersLastRedir) as $line) {
125 if (empty($line) or ctype_space($line)) {
126 continue;
127 }
128 $splitLine = explode(': ', $line, 2);
129 if (count($splitLine) > 1) {
130 $key = $splitLine[0];
131 $value = $splitLine[1];
132 if (array_key_exists($key, $headers)) {
133 if (!is_array($headers[$key])) {
134 $headers[$key] = array(0 => $headers[$key]);
135 }
136 $headers[$key][] = $value;
137 } else {
138 $headers[$key] = $value;
139 }
140 } else {
141 $headers[] = $splitLine[0];
142 }
143 }
144
145 return array($headers, $content);
146 }
147
148 /**
149 * GET an HTTP URL to retrieve its content (fallback method)
150 *
151 * @param string $cleanUrl URL to get (http://... valid and in ASCII form)
152 * @param int $timeout network timeout (in seconds)
153 * @param int $maxBytes maximum downloaded bytes
154 * @param string $userAgent "User-Agent" header
155 * @param string $acceptLanguage "Accept-Language" header
156 * @param int $maxRedr maximum amount of redirections followed
157 *
158 * @return array HTTP response headers, downloaded content
159 *
160 * Output format:
161 * [0] = associative array containing HTTP response headers
162 * [1] = URL content (downloaded data)
163 *
164 * @see http://php.net/manual/en/function.file-get-contents.php
165 * @see http://php.net/manual/en/function.stream-context-create.php
166 * @see http://php.net/manual/en/function.get-headers.php
167 */
168 function get_http_response_fallback(
169 $cleanUrl,
170 $timeout,
171 $maxBytes,
172 $userAgent,
173 $acceptLanguage,
174 $maxRedr
175 ) {
176 $options = array(
177 'http' => array(
178 'method' => 'GET',
179 'timeout' => $timeout,
180 'user_agent' => $userAgent,
181 'header' => "Accept: */*\r\n"
182 . 'Accept-Language: ' . $acceptLanguage
183 )
184 );
185
186 stream_context_set_default($options);
187 list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
188 if (! $headers || strpos($headers[0], '200 OK') === false) {
189 $options['http']['request_fulluri'] = true;
190 stream_context_set_default($options);
191 list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
192 }
193
194 if (! $headers) {
195 return array($headers, false);
196 }
197
198 try {
199 // TODO: catch Exception in calling code (thumbnailer)
200 $context = stream_context_create($options);
201 $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
202 } catch (Exception $exc) {
203 return array(array(0 => 'HTTP Error'), $exc->getMessage());
204 }
205
206 return array($headers, $content);
207 }
208
209 /**
210 * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
211 *
212 * @param string $url initial URL to reach.
213 * @param int $redirectionLimit max redirection follow.
214 *
215 * @return array HTTP headers, or false if it failed.
216 */
217 function get_redirected_headers($url, $redirectionLimit = 3)
218 {
219 $headers = get_headers($url, 1);
220 if (!empty($headers['location']) && empty($headers['Location'])) {
221 $headers['Location'] = $headers['location'];
222 }
223
224 // Headers found, redirection found, and limit not reached.
225 if ($redirectionLimit-- > 0
226 && !empty($headers)
227 && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
228 && !empty($headers['Location'])) {
229
230 $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
231 if ($redirection != $url) {
232 $redirection = getAbsoluteUrl($url, $redirection);
233 return get_redirected_headers($redirection, $redirectionLimit);
234 }
235 }
236
237 return array($headers, $url);
238 }
239
240 /**
241 * Get an absolute URL from a complete one, and another absolute/relative URL.
242 *
243 * @param string $originalUrl The original complete URL.
244 * @param string $newUrl The new one, absolute or relative.
245 *
246 * @return string Final URL:
247 * - $newUrl if it was already an absolute URL.
248 * - if it was relative, absolute URL from $originalUrl path.
249 */
250 function getAbsoluteUrl($originalUrl, $newUrl)
251 {
252 $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
253 // Already an absolute URL.
254 if (!empty($newScheme)) {
255 return $newUrl;
256 }
257
258 $parts = parse_url($originalUrl);
259 $final = $parts['scheme'] .'://'. $parts['host'];
260 $final .= (!empty($parts['port'])) ? $parts['port'] : '';
261 $final .= '/';
262 if ($newUrl[0] != '/') {
263 $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
264 }
265 $final .= ltrim($newUrl, '/');
266 return $final;
267 }
268
269 /**
270 * Returns the server's base URL: scheme://domain.tld[:port]
271 *
272 * @param array $server the $_SERVER array
273 *
274 * @return string the server's base URL
275 *
276 * @see http://www.ietf.org/rfc/rfc7239.txt
277 * @see http://www.ietf.org/rfc/rfc6648.txt
278 * @see http://stackoverflow.com/a/3561399
279 * @see http://stackoverflow.com/q/452375
280 */
281 function server_url($server)
282 {
283 $scheme = 'http';
284 $port = '';
285
286 // Shaarli is served behind a proxy
287 if (isset($server['HTTP_X_FORWARDED_PROTO'])) {
288 // Keep forwarded scheme
289 if (strpos($server['HTTP_X_FORWARDED_PROTO'], ',') !== false) {
290 $schemes = explode(',', $server['HTTP_X_FORWARDED_PROTO']);
291 $scheme = trim($schemes[0]);
292 } else {
293 $scheme = $server['HTTP_X_FORWARDED_PROTO'];
294 }
295
296 if (isset($server['HTTP_X_FORWARDED_PORT'])) {
297 // Keep forwarded port
298 if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
299 $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
300 $port = ':' . trim($ports[0]);
301 } else {
302 $port = ':' . $server['HTTP_X_FORWARDED_PORT'];
303 }
304 }
305
306 return $scheme.'://'.$server['SERVER_NAME'].$port;
307 }
308
309 // SSL detection
310 if ((! empty($server['HTTPS']) && strtolower($server['HTTPS']) == 'on')
311 || (isset($server['SERVER_PORT']) && $server['SERVER_PORT'] == '443')) {
312 $scheme = 'https';
313 }
314
315 // Do not append standard port values
316 if (($scheme == 'http' && $server['SERVER_PORT'] != '80')
317 || ($scheme == 'https' && $server['SERVER_PORT'] != '443')) {
318 $port = ':'.$server['SERVER_PORT'];
319 }
320
321 return $scheme.'://'.$server['SERVER_NAME'].$port;
322 }
323
324 /**
325 * Returns the absolute URL of the current script, without the query
326 *
327 * If the resource is "index.php", then it is removed (for better-looking URLs)
328 *
329 * @param array $server the $_SERVER array
330 *
331 * @return string the absolute URL of the current script, without the query
332 */
333 function index_url($server)
334 {
335 $scriptname = $server['SCRIPT_NAME'];
336 if (endsWith($scriptname, 'index.php')) {
337 $scriptname = substr($scriptname, 0, -9);
338 }
339 return server_url($server) . $scriptname;
340 }
341
342 /**
343 * Returns the absolute URL of the current script, with the query
344 *
345 * If the resource is "index.php", then it is removed (for better-looking URLs)
346 *
347 * @param array $server the $_SERVER array
348 *
349 * @return string the absolute URL of the current script, with the query
350 */
351 function page_url($server)
352 {
353 if (! empty($server['QUERY_STRING'])) {
354 return index_url($server).'?'.$server['QUERY_STRING'];
355 }
356 return index_url($server);
357 }
358
359 /**
360 * Retrieve the initial IP forwarded by the reverse proxy.
361 *
362 * Inspired from: https://github.com/zendframework/zend-http/blob/master/src/PhpEnvironment/RemoteAddress.php
363 *
364 * @param array $server $_SERVER array which contains HTTP headers.
365 * @param array $trustedIps List of trusted IP from the configuration.
366 *
367 * @return string|bool The forwarded IP, or false if none could be extracted.
368 */
369 function getIpAddressFromProxy($server, $trustedIps)
370 {
371 $forwardedIpHeader = 'HTTP_X_FORWARDED_FOR';
372 if (empty($server[$forwardedIpHeader])) {
373 return false;
374 }
375
376 $ips = preg_split('/\s*,\s*/', $server[$forwardedIpHeader]);
377 $ips = array_diff($ips, $trustedIps);
378 if (empty($ips)) {
379 return false;
380 }
381
382 return array_pop($ips);
383 }