3 * GET an HTTP URL to retrieve its content
4 * Uses the cURL library or a fallback method
6 * @param string $url URL to get (http://...)
7 * @param int $timeout network timeout (in seconds)
8 * @param int $maxBytes maximum downloaded bytes (default: 4 MiB)
9 * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
10 * Can be used to add download conditions on the
11 * headers (response code, content type, etc.).
13 * @return array HTTP response headers, downloaded content
16 * [0] = associative array containing HTTP response headers
17 * [1] = URL content (downloaded data)
20 * list($headers, $data) = get_http_response('http://sebauvage.net/');
21 * if (strpos($headers[0], '200 OK') !== false) {
22 * echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
24 * echo 'There was an error: '.htmlspecialchars($headers[0]);
27 * @see https://secure.php.net/manual/en/ref.curl.php
28 * @see https://secure.php.net/manual/en/functions.anonymous.php
29 * @see https://secure.php.net/manual/en/function.preg-split.php
30 * @see https://secure.php.net/manual/en/function.explode.php
31 * @see http://stackoverflow.com/q/17641073
32 * @see http://stackoverflow.com/q/9183178
33 * @see http://stackoverflow.com/q/1462720
35 function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null)
37 $urlObj = new Url($url);
38 $cleanUrl = $urlObj->idnToAscii();
40 if (!filter_var($cleanUrl, FILTER_VALIDATE_URL
) || !$urlObj->isHttp()) {
41 return array(array(0 => 'Invalid HTTP Url'), false);
45 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
46 . ' Gecko/20100101 Firefox/45.0';
48 substr(setlocale(LC_COLLATE
, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3';
51 if (!function_exists('curl_init')) {
52 return get_http_response_fallback(
62 $ch = curl_init($cleanUrl);
64 return array(array(0 => 'curl_init() error'), false);
67 // General cURL settings
68 curl_setopt($ch, CURLOPT_AUTOREFERER
, true);
69 curl_setopt($ch, CURLOPT_FOLLOWLOCATION
, true);
70 curl_setopt($ch, CURLOPT_HEADER
, true);
74 array('Accept-Language: ' . $acceptLanguage)
76 curl_setopt($ch, CURLOPT_MAXREDIRS
, $maxRedirs);
77 curl_setopt($ch, CURLOPT_RETURNTRANSFER
, true);
78 curl_setopt($ch, CURLOPT_TIMEOUT
, $timeout);
79 curl_setopt($ch, CURLOPT_USERAGENT
, $userAgent);
81 if (is_callable($curlWriteFunction)) {
82 curl_setopt($ch, CURLOPT_WRITEFUNCTION
, $curlWriteFunction);
85 // Max download size management
86 curl_setopt($ch, CURLOPT_BUFFERSIZE
, 1024*16);
87 curl_setopt($ch, CURLOPT_NOPROGRESS
, false);
90 CURLOPT_PROGRESSFUNCTION
,
91 function ($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes) {
92 if (version_compare(phpversion(), '5.5', '<')) {
93 // PHP version lower than 5.5
94 // Callback has 4 arguments
97 // Callback has 5 arguments
100 // Non-zero return stops downloading
101 return ($downloaded > $maxBytes) ? 1 : 0;
105 $response = curl_exec($ch);
106 $errorNo = curl_errno($ch);
107 $errorStr = curl_error($ch);
108 $headSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE
);
111 if ($response === false) {
112 if ($errorNo == CURLE_COULDNT_RESOLVE_HOST
) {
114 * Workaround to match fallback method behaviour
115 * Removing this would require updating
116 * GetHttpUrlTest::testGetInvalidRemoteUrl()
118 return array(false, false);
120 return array(array(0 => 'curl_exec() error: ' . $errorStr), false);
123 // Formatting output like the fallback method
124 $rawHeaders = substr($response, 0, $headSize);
126 // Keep only headers from latest redirection
127 $rawHeadersArrayRedirs = explode("\r\n\r\n", trim($rawHeaders));
128 $rawHeadersLastRedir = end($rawHeadersArrayRedirs);
130 $content = substr($response, $headSize);
132 foreach (preg_split('~[\r\n]+~', $rawHeadersLastRedir) as $line) {
133 if (empty($line) || ctype_space($line)) {
136 $splitLine = explode(': ', $line, 2);
137 if (count($splitLine) > 1) {
138 $key = $splitLine[0];
139 $value = $splitLine[1];
140 if (array_key_exists($key, $headers)) {
141 if (!is_array($headers[$key])) {
142 $headers[$key] = array(0 => $headers[$key]);
144 $headers[$key][] = $value;
146 $headers[$key] = $value;
149 $headers[] = $splitLine[0];
153 return array($headers, $content);
157 * GET an HTTP URL to retrieve its content (fallback method)
159 * @param string $cleanUrl URL to get (http://... valid and in ASCII form)
160 * @param int $timeout network timeout (in seconds)
161 * @param int $maxBytes maximum downloaded bytes
162 * @param string $userAgent "User-Agent" header
163 * @param string $acceptLanguage "Accept-Language" header
164 * @param int $maxRedr maximum amount of redirections followed
166 * @return array HTTP response headers, downloaded content
169 * [0] = associative array containing HTTP response headers
170 * [1] = URL content (downloaded data)
172 * @see http://php.net/manual/en/function.file-get-contents.php
173 * @see http://php.net/manual/en/function.stream-context-create.php
174 * @see http://php.net/manual/en/function.get-headers.php
176 function get_http_response_fallback(
187 'timeout' => $timeout,
188 'user_agent' => $userAgent,
189 'header' => "Accept: */*\r\n"
190 . 'Accept-Language: ' . $acceptLanguage
194 stream_context_set_default($options);
195 list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
196 if (! $headers || strpos($headers[0], '200 OK') === false) {
197 $options['http']['request_fulluri'] = true;
198 stream_context_set_default($options);
199 list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
203 return array($headers, false);
207 // TODO: catch Exception in calling code (thumbnailer)
208 $context = stream_context_create($options);
209 $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
210 } catch (Exception
$exc) {
211 return array(array(0 => 'HTTP Error'), $exc->getMessage());
214 return array($headers, $content);
218 * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
220 * @param string $url initial URL to reach.
221 * @param int $redirectionLimit max redirection follow.
223 * @return array HTTP headers, or false if it failed.
225 function get_redirected_headers($url, $redirectionLimit = 3)
227 $headers = get_headers($url, 1);
228 if (!empty($headers['location']) && empty($headers['Location'])) {
229 $headers['Location'] = $headers['location'];
232 // Headers found, redirection found, and limit not reached.
233 if ($redirectionLimit-- > 0
235 && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
236 && !empty($headers['Location'])) {
237 $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
238 if ($redirection != $url) {
239 $redirection = getAbsoluteUrl($url, $redirection);
240 return get_redirected_headers($redirection, $redirectionLimit);
244 return array($headers, $url);
248 * Get an absolute URL from a complete one, and another absolute/relative URL.
250 * @param string $originalUrl The original complete URL.
251 * @param string $newUrl The new one, absolute or relative.
253 * @return string Final URL:
254 * - $newUrl if it was already an absolute URL.
255 * - if it was relative, absolute URL from $originalUrl path.
257 function getAbsoluteUrl($originalUrl, $newUrl)
259 $newScheme = parse_url($newUrl, PHP_URL_SCHEME
);
260 // Already an absolute URL.
261 if (!empty($newScheme)) {
265 $parts = parse_url($originalUrl);
266 $final = $parts['scheme'] .'://'. $parts['host'];
267 $final .= (!empty($parts['port'])) ? $parts['port'] : '';
269 if ($newUrl[0] != '/') {
270 $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
272 $final .= ltrim($newUrl, '/');
277 * Returns the server's base URL: scheme://domain.tld[:port]
279 * @param array $server the $_SERVER array
281 * @return string the server's base URL
283 * @see http://www.ietf.org/rfc/rfc7239.txt
284 * @see http://www.ietf.org/rfc/rfc6648.txt
285 * @see http://stackoverflow.com/a/3561399
286 * @see http://stackoverflow.com/q/452375
288 function server_url($server)
293 // Shaarli is served behind a proxy
294 if (isset($server['HTTP_X_FORWARDED_PROTO'])) {
295 // Keep forwarded scheme
296 if (strpos($server['HTTP_X_FORWARDED_PROTO'], ',') !== false) {
297 $schemes = explode(',', $server['HTTP_X_FORWARDED_PROTO']);
298 $scheme = trim($schemes[0]);
300 $scheme = $server['HTTP_X_FORWARDED_PROTO'];
303 if (isset($server['HTTP_X_FORWARDED_PORT'])) {
304 // Keep forwarded port
305 if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
306 $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
307 $port = trim($ports[0]);
309 $port = $server['HTTP_X_FORWARDED_PORT'];
312 // This is a workaround for proxies that don't forward the scheme properly.
313 // Connecting over port 443 has to be in HTTPS.
314 // See https://github.com/shaarli/Shaarli/issues/1022
315 if ($port == '443') {
319 if (($scheme == 'http' && $port != '80')
320 || ($scheme == 'https' && $port != '443')
328 if (isset($server['HTTP_X_FORWARDED_HOST'])) {
329 // Keep forwarded host
330 if (strpos($server['HTTP_X_FORWARDED_HOST'], ',') !== false) {
331 $hosts = explode(',', $server['HTTP_X_FORWARDED_HOST']);
332 $host = trim($hosts[0]);
334 $host = $server['HTTP_X_FORWARDED_HOST'];
337 $host = $server['SERVER_NAME'];
340 return $scheme.'://'.$host.$port;
344 if ((! empty($server['HTTPS']) && strtolower($server['HTTPS']) == 'on')
345 || (isset($server['SERVER_PORT']) && $server['SERVER_PORT'] == '443')) {
349 // Do not append standard port values
350 if (($scheme == 'http' && $server['SERVER_PORT'] != '80')
351 || ($scheme == 'https' && $server['SERVER_PORT'] != '443')) {
352 $port = ':'.$server['SERVER_PORT'];
355 return $scheme.'://'.$server['SERVER_NAME'].$port;
359 * Returns the absolute URL of the current script, without the query
361 * If the resource is "index.php", then it is removed (for better-looking URLs)
363 * @param array $server the $_SERVER array
365 * @return string the absolute URL of the current script, without the query
367 function index_url($server)
369 $scriptname = $server['SCRIPT_NAME'];
370 if (endsWith($scriptname, 'index.php')) {
371 $scriptname = substr($scriptname, 0, -9);
373 return server_url($server) . $scriptname;
377 * Returns the absolute URL of the current script, with the query
379 * If the resource is "index.php", then it is removed (for better-looking URLs)
381 * @param array $server the $_SERVER array
383 * @return string the absolute URL of the current script, with the query
385 function page_url($server)
387 if (! empty($server['QUERY_STRING'])) {
388 return index_url($server).'?'.$server['QUERY_STRING'];
390 return index_url($server);
394 * Retrieve the initial IP forwarded by the reverse proxy.
396 * Inspired from: https://github.com/zendframework/zend-http/blob/master/src/PhpEnvironment/RemoteAddress.php
398 * @param array $server $_SERVER array which contains HTTP headers.
399 * @param array $trustedIps List of trusted IP from the configuration.
401 * @return string|bool The forwarded IP, or false if none could be extracted.
403 function getIpAddressFromProxy($server, $trustedIps)
405 $forwardedIpHeader = 'HTTP_X_FORWARDED_FOR';
406 if (empty($server[$forwardedIpHeader])) {
410 $ips = preg_split('/\s*,\s*/', $server[$forwardedIpHeader]);
411 $ips = array_diff($ips, $trustedIps);
416 return array_pop($ips);
421 * Return an identifier based on the advertised client IP address(es)
423 * This aims at preventing session hijacking from users behind the same proxy
424 * by relying on HTTP headers.
427 * - https://secure.php.net/manual/en/reserved.variables.server.php
428 * - https://stackoverflow.com/questions/3003145/how-to-get-the-client-ip-address-in-php
429 * - https://stackoverflow.com/questions/12233406/preventing-session-hijacking
430 * - https://stackoverflow.com/questions/21354859/trusting-x-forwarded-for-to-identify-a-visitor
432 * @param array $server The $_SERVER array
434 * @return string An identifier based on client IP address information
436 function client_ip_id($server)
438 $ip = $server['REMOTE_ADDR'];
440 if (isset($server['HTTP_X_FORWARDED_FOR'])) {
441 $ip = $ip . '_' . $server['HTTP_X_FORWARDED_FOR'];
443 if (isset($server['HTTP_CLIENT_IP'])) {
444 $ip = $ip . '_' . $server['HTTP_CLIENT_IP'];
451 * Returns true if Shaarli's currently browsed in HTTPS.
452 * Supports reverse proxies (if the headers are correctly set).
454 * @param array $server $_SERVER.
456 * @return bool true if HTTPS, false otherwise.
458 function is_https($server)
461 if (isset($server['HTTP_X_FORWARDED_PORT'])) {
462 // Keep forwarded port
463 if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
464 $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
465 $port = trim($ports[0]);
467 $port = $server['HTTP_X_FORWARDED_PORT'];
470 if ($port == '443') {
475 return ! empty($server['HTTPS']);