application/HttpUtils.php

   1 <?php
   2 /**
   3  * GET an HTTP URL to retrieve its content
   4  * Uses the cURL library or a fallback method
   5  *
   6  * @param string $url      URL to get (http://...)
   7  * @param int    $timeout  network timeout (in seconds)
   8  * @param int    $maxBytes maximum downloaded bytes (default: 4 MiB)
   9  *
  10  * @return array HTTP response headers, downloaded content
  11  *
  12  * Output format:
  13  *  [0] = associative array containing HTTP response headers
  14  *  [1] = URL content (downloaded data)
  15  *
  16  * Example:
  17  *  list($headers, $data) = get_http_response('http://sebauvage.net/');
  18  *  if (strpos($headers[0], '200 OK') !== false) {
  19  *      echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
  20  *  } else {
  21  *      echo 'There was an error: '.htmlspecialchars($headers[0]);
  22  *  }
  23  *
  24  * @see https://secure.php.net/manual/en/ref.curl.php
  25  * @see https://secure.php.net/manual/en/functions.anonymous.php
  26  * @see https://secure.php.net/manual/en/function.preg-split.php
  27  * @see https://secure.php.net/manual/en/function.explode.php
  28  * @see http://stackoverflow.com/q/17641073
  29  * @see http://stackoverflow.com/q/9183178
  30  * @see http://stackoverflow.com/q/1462720
  31  */
  32 function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
  33 {
  34     $urlObj = new Url($url);
  35     $cleanUrl = $urlObj->idnToAscii();
  36
  37     if (!filter_var($cleanUrl, FILTER_VALIDATE_URL) || !$urlObj->isHttp()) {
  38         return array(array(0 => 'Invalid HTTP Url'), false);
  39     }
  40
  41     $userAgent =
  42         'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
  43         . ' Gecko/20100101 Firefox/45.0';
  44     $acceptLanguage =
  45         substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3';
  46     $maxRedirs = 3;
  47
  48     if (!function_exists('curl_init')) {
  49         return get_http_response_fallback(
  50             $cleanUrl,
  51             $timeout,
  52             $maxBytes,
  53             $userAgent,
  54             $acceptLanguage,
  55             $maxRedirs
  56         );
  57     }
  58
  59     $ch = curl_init($cleanUrl);
  60     if ($ch === false) {
  61         return array(array(0 => 'curl_init() error'), false);
  62     }
  63
  64     // General cURL settings
  65     curl_setopt($ch, CURLOPT_AUTOREFERER,       true);
  66     curl_setopt($ch, CURLOPT_FOLLOWLOCATION,    true);
  67     curl_setopt($ch, CURLOPT_HEADER,            true);
  68     curl_setopt(
  69         $ch,
  70         CURLOPT_HTTPHEADER,
  71         array('Accept-Language: ' . $acceptLanguage)
  72     );
  73     curl_setopt($ch, CURLOPT_MAXREDIRS,         $maxRedirs);
  74     curl_setopt($ch, CURLOPT_RETURNTRANSFER,    true);
  75     curl_setopt($ch, CURLOPT_TIMEOUT,           $timeout);
  76     curl_setopt($ch, CURLOPT_USERAGENT,         $userAgent);
  77
  78     // Max download size management
  79     curl_setopt($ch, CURLOPT_BUFFERSIZE,        1024);
  80     curl_setopt($ch, CURLOPT_NOPROGRESS,        false);
  81     curl_setopt($ch, CURLOPT_PROGRESSFUNCTION,
  82         function($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes)
  83         {
  84             if (version_compare(phpversion(), '5.5', '<')) {
  85                 // PHP version lower than 5.5
  86                 // Callback has 4 arguments
  87                 $downloaded = $arg1;
  88             } else {
  89                 // Callback has 5 arguments
  90                 $downloaded = $arg2;
  91             }
  92             // Non-zero return stops downloading
  93             return ($downloaded > $maxBytes) ? 1 : 0;
  94         }
  95     );
  96
  97     $response = curl_exec($ch);
  98     $errorNo = curl_errno($ch);
  99     $errorStr = curl_error($ch);
 100     $headSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
 101     curl_close($ch);
 102
 103     if ($response === false) {
 104         if ($errorNo == CURLE_COULDNT_RESOLVE_HOST) {
 105             /*
 106              * Workaround to match fallback method behaviour
 107              * Removing this would require updating
 108              * GetHttpUrlTest::testGetInvalidRemoteUrl()
 109              */
 110             return array(false, false);
 111         }
 112         return array(array(0 => 'curl_exec() error: ' . $errorStr), false);
 113     }
 114
 115     // Formatting output like the fallback method
 116     $rawHeaders = substr($response, 0, $headSize);
 117
 118     // Keep only headers from latest redirection
 119     $rawHeadersArrayRedirs = explode("\r\n\r\n", trim($rawHeaders));
 120     $rawHeadersLastRedir = end($rawHeadersArrayRedirs);
 121
 122     $content = substr($response, $headSize);
 123     $headers = array();
 124     foreach (preg_split('~[\r\n]+~', $rawHeadersLastRedir) as $line) {
 125         if (empty($line) || ctype_space($line)) {
 126             continue;
 127         }
 128         $splitLine = explode(': ', $line, 2);
 129         if (count($splitLine) > 1) {
 130             $key = $splitLine[0];
 131             $value = $splitLine[1];
 132             if (array_key_exists($key, $headers)) {
 133                 if (!is_array($headers[$key])) {
 134                     $headers[$key] = array(0 => $headers[$key]);
 135                 }
 136                 $headers[$key][] = $value;
 137             } else {
 138                 $headers[$key] = $value;
 139             }
 140         } else {
 141             $headers[] = $splitLine[0];
 142         }
 143     }
 144
 145     return array($headers, $content);
 146 }
 147
 148 /**
 149  * GET an HTTP URL to retrieve its content (fallback method)
 150  *
 151  * @param string $cleanUrl       URL to get (http://... valid and in ASCII form)
 152  * @param int    $timeout        network timeout (in seconds)
 153  * @param int    $maxBytes       maximum downloaded bytes
 154  * @param string $userAgent      "User-Agent" header
 155  * @param string $acceptLanguage "Accept-Language" header
 156  * @param int    $maxRedr        maximum amount of redirections followed
 157  *
 158  * @return array HTTP response headers, downloaded content
 159  *
 160  * Output format:
 161  *  [0] = associative array containing HTTP response headers
 162  *  [1] = URL content (downloaded data)
 163  *
 164  * @see http://php.net/manual/en/function.file-get-contents.php
 165  * @see http://php.net/manual/en/function.stream-context-create.php
 166  * @see http://php.net/manual/en/function.get-headers.php
 167  */
 168 function get_http_response_fallback(
 169     $cleanUrl,
 170     $timeout,
 171     $maxBytes,
 172     $userAgent,
 173     $acceptLanguage,
 174     $maxRedr
 175 ) {
 176     $options = array(
 177         'http' => array(
 178             'method' => 'GET',
 179             'timeout' => $timeout,
 180             'user_agent' => $userAgent,
 181             'header' => "Accept: */*\r\n"
 182                 . 'Accept-Language: ' . $acceptLanguage
 183         )
 184     );
 185
 186     stream_context_set_default($options);
 187     list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
 188     if (! $headers || strpos($headers[0], '200 OK') === false) {
 189         $options['http']['request_fulluri'] = true;
 190         stream_context_set_default($options);
 191         list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
 192     }
 193
 194     if (! $headers) {
 195         return array($headers, false);
 196     }
 197
 198     try {
 199         // TODO: catch Exception in calling code (thumbnailer)
 200         $context = stream_context_create($options);
 201         $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
 202     } catch (Exception $exc) {
 203         return array(array(0 => 'HTTP Error'), $exc->getMessage());
 204     }
 205
 206     return array($headers, $content);
 207 }
 208
 209 /**
 210  * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
 211  *
 212  * @param string $url              initial URL to reach.
 213  * @param int    $redirectionLimit max redirection follow.
 214  *
 215  * @return array HTTP headers, or false if it failed.
 216  */
 217 function get_redirected_headers($url, $redirectionLimit = 3)
 218 {
 219     $headers = get_headers($url, 1);
 220     if (!empty($headers['location']) && empty($headers['Location'])) {
 221         $headers['Location'] = $headers['location'];
 222     }
 223
 224     // Headers found, redirection found, and limit not reached.
 225     if ($redirectionLimit-- > 0
 226         && !empty($headers)
 227         && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
 228         && !empty($headers['Location'])) {
 229
 230         $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
 231         if ($redirection != $url) {
 232             $redirection = getAbsoluteUrl($url, $redirection);
 233             return get_redirected_headers($redirection, $redirectionLimit);
 234         }
 235     }
 236
 237     return array($headers, $url);
 238 }
 239
 240 /**
 241  * Get an absolute URL from a complete one, and another absolute/relative URL.
 242  *
 243  * @param string $originalUrl The original complete URL.
 244  * @param string $newUrl      The new one, absolute or relative.
 245  *
 246  * @return string Final URL:
 247  *   - $newUrl if it was already an absolute URL.
 248  *   - if it was relative, absolute URL from $originalUrl path.
 249  */
 250 function getAbsoluteUrl($originalUrl, $newUrl)
 251 {
 252     $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
 253     // Already an absolute URL.
 254     if (!empty($newScheme)) {
 255         return $newUrl;
 256     }
 257
 258     $parts = parse_url($originalUrl);
 259     $final = $parts['scheme'] .'://'. $parts['host'];
 260     $final .= (!empty($parts['port'])) ? $parts['port'] : '';
 261     $final .= '/';
 262     if ($newUrl[0] != '/') {
 263         $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
 264     }
 265     $final .= ltrim($newUrl, '/');
 266     return $final;
 267 }
 268
 269 /**
 270  * Returns the server's base URL: scheme://domain.tld[:port]
 271  *
 272  * @param array $server the $_SERVER array
 273  *
 274  * @return string the server's base URL
 275  *
 276  * @see http://www.ietf.org/rfc/rfc7239.txt
 277  * @see http://www.ietf.org/rfc/rfc6648.txt
 278  * @see http://stackoverflow.com/a/3561399
 279  * @see http://stackoverflow.com/q/452375
 280  */
 281 function server_url($server)
 282 {
 283     $scheme = 'http';
 284     $port = '';
 285
 286     // Shaarli is served behind a proxy
 287     if (isset($server['HTTP_X_FORWARDED_PROTO'])) {
 288         // Keep forwarded scheme
 289         if (strpos($server['HTTP_X_FORWARDED_PROTO'], ',') !== false) {
 290             $schemes = explode(',', $server['HTTP_X_FORWARDED_PROTO']);
 291             $scheme = trim($schemes[0]);
 292         } else {
 293             $scheme = $server['HTTP_X_FORWARDED_PROTO'];
 294         }
 295
 296         if (isset($server['HTTP_X_FORWARDED_PORT'])) {
 297             // Keep forwarded port
 298             if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
 299                 $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
 300                 $port = trim($ports[0]);
 301             } else {
 302                 $port = $server['HTTP_X_FORWARDED_PORT'];
 303             }
 304
 305             // This is a workaround for proxies that don't forward the scheme properly.
 306             // Connecting over port 443 has to be in HTTPS.
 307             // See https://github.com/shaarli/Shaarli/issues/1022
 308             if ($port == '443') {
 309                 $scheme = 'https';
 310             }
 311
 312             if (($scheme == 'http' && $port != '80')
 313                 || ($scheme == 'https' && $port != '443')
 314             ) {
 315                 $port = ':' . $port;
 316             } else {
 317                 $port = '';
 318             }
 319         }
 320
 321         if (isset($server['HTTP_X_FORWARDED_HOST'])) {
 322             // Keep forwarded host
 323             if (strpos($server['HTTP_X_FORWARDED_HOST'], ',') !== false) {
 324                 $hosts = explode(',', $server['HTTP_X_FORWARDED_HOST']);
 325                 $host = trim($hosts[0]);
 326             } else {
 327                 $host = $server['HTTP_X_FORWARDED_HOST'];
 328             }
 329         } else {
 330             $host = $server['SERVER_NAME'];
 331         }
 332
 333         return $scheme.'://'.$host.$port;
 334     }
 335
 336     // SSL detection
 337     if ((! empty($server['HTTPS']) && strtolower($server['HTTPS']) == 'on')
 338         || (isset($server['SERVER_PORT']) && $server['SERVER_PORT'] == '443')) {
 339         $scheme = 'https';
 340     }
 341
 342     // Do not append standard port values
 343     if (($scheme == 'http' && $server['SERVER_PORT'] != '80')
 344         || ($scheme == 'https' && $server['SERVER_PORT'] != '443')) {
 345         $port = ':'.$server['SERVER_PORT'];
 346     }
 347
 348     return $scheme.'://'.$server['SERVER_NAME'].$port;
 349 }
 350
 351 /**
 352  * Returns the absolute URL of the current script, without the query
 353  *
 354  * If the resource is "index.php", then it is removed (for better-looking URLs)
 355  *
 356  * @param array $server the $_SERVER array
 357  *
 358  * @return string the absolute URL of the current script, without the query
 359  */
 360 function index_url($server)
 361 {
 362     $scriptname = $server['SCRIPT_NAME'];
 363     if (endsWith($scriptname, 'index.php')) {
 364         $scriptname = substr($scriptname, 0, -9);
 365     }
 366     return server_url($server) . $scriptname;
 367 }
 368
 369 /**
 370  * Returns the absolute URL of the current script, with the query
 371  *
 372  * If the resource is "index.php", then it is removed (for better-looking URLs)
 373  *
 374  * @param array $server the $_SERVER array
 375  *
 376  * @return string the absolute URL of the current script, with the query
 377  */
 378 function page_url($server)
 379 {
 380     if (! empty($server['QUERY_STRING'])) {
 381         return index_url($server).'?'.$server['QUERY_STRING'];
 382     }
 383     return index_url($server);
 384 }
 385
 386 /**
 387  * Retrieve the initial IP forwarded by the reverse proxy.
 388  *
 389  * Inspired from: https://github.com/zendframework/zend-http/blob/master/src/PhpEnvironment/RemoteAddress.php
 390  *
 391  * @param array $server     $_SERVER array which contains HTTP headers.
 392  * @param array $trustedIps List of trusted IP from the configuration.
 393  *
 394  * @return string|bool The forwarded IP, or false if none could be extracted.
 395  */
 396 function getIpAddressFromProxy($server, $trustedIps)
 397 {
 398     $forwardedIpHeader = 'HTTP_X_FORWARDED_FOR';
 399     if (empty($server[$forwardedIpHeader])) {
 400         return false;
 401     }
 402
 403     $ips = preg_split('/\s*,\s*/', $server[$forwardedIpHeader]);
 404     $ips = array_diff($ips, $trustedIps);
 405     if (empty($ips)) {
 406         return false;
 407     }
 408
 409     return array_pop($ips);
 410 }
 411
 412 /**
 413  * Returns true if Shaarli's currently browsed in HTTPS.
 414  * Supports reverse proxies (if the headers are correctly set).
 415  *
 416  * @param array $server $_SERVER.
 417  *
 418  * @return bool true if HTTPS, false otherwise.
 419  */
 420 function is_https($server)
 421 {
 422
 423     if (isset($server['HTTP_X_FORWARDED_PORT'])) {
 424         // Keep forwarded port
 425         if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
 426             $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
 427             $port = trim($ports[0]);
 428         } else {
 429             $port = $server['HTTP_X_FORWARDED_PORT'];
 430         }
 431
 432         if ($port == '443') {
 433             return true;
 434         }
 435     }
 436
 437     return ! empty($server['HTTPS']);
 438 }