HttpUtils.php

   1 <?php
   2 /**
   3  * GET an HTTP URL to retrieve its content
   4  * Uses the cURL library or a fallback method
   5  *
   6  * @param string          $url               URL to get (http://...)
   7  * @param int             $timeout           network timeout (in seconds)
   8  * @param int             $maxBytes          maximum downloaded bytes (default: 4 MiB)
   9  * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
  10  *                                           Can be used to add download conditions on the headers (response code, content type, etc.).
  11  *
  12  * @return array HTTP response headers, downloaded content
  13  *
  14  * Output format:
  15  *  [0] = associative array containing HTTP response headers
  16  *  [1] = URL content (downloaded data)
  17  *
  18  * Example:
  19  *  list($headers, $data) = get_http_response('http://sebauvage.net/');
  20  *  if (strpos($headers[0], '200 OK') !== false) {
  21  *      echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
  22  *  } else {
  23  *      echo 'There was an error: '.htmlspecialchars($headers[0]);
  24  *  }
  25  *
  26  * @see https://secure.php.net/manual/en/ref.curl.php
  27  * @see https://secure.php.net/manual/en/functions.anonymous.php
  28  * @see https://secure.php.net/manual/en/function.preg-split.php
  29  * @see https://secure.php.net/manual/en/function.explode.php
  30  * @see http://stackoverflow.com/q/17641073
  31  * @see http://stackoverflow.com/q/9183178
  32  * @see http://stackoverflow.com/q/1462720
  33  */
  34 function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null)
  35 {
  36     $urlObj = new Url($url);
  37     $cleanUrl = $urlObj->idnToAscii();
  38
  39     if (!filter_var($cleanUrl, FILTER_VALIDATE_URL) || !$urlObj->isHttp()) {
  40         return array(array(0 => 'Invalid HTTP Url'), false);
  41     }
  42
  43     $userAgent =
  44         'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
  45         . ' Gecko/20100101 Firefox/45.0';
  46     $acceptLanguage =
  47         substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3';
  48     $maxRedirs = 3;
  49
  50     if (!function_exists('curl_init')) {
  51         return get_http_response_fallback(
  52             $cleanUrl,
  53             $timeout,
  54             $maxBytes,
  55             $userAgent,
  56             $acceptLanguage,
  57             $maxRedirs
  58         );
  59     }
  60
  61     $ch = curl_init($cleanUrl);
  62     if ($ch === false) {
  63         return array(array(0 => 'curl_init() error'), false);
  64     }
  65
  66     // General cURL settings
  67     curl_setopt($ch, CURLOPT_AUTOREFERER,       true);
  68     curl_setopt($ch, CURLOPT_FOLLOWLOCATION,    true);
  69     curl_setopt($ch, CURLOPT_HEADER,            true);
  70     curl_setopt(
  71         $ch,
  72         CURLOPT_HTTPHEADER,
  73         array('Accept-Language: ' . $acceptLanguage)
  74     );
  75     curl_setopt($ch, CURLOPT_MAXREDIRS,         $maxRedirs);
  76     curl_setopt($ch, CURLOPT_RETURNTRANSFER,    true);
  77     curl_setopt($ch, CURLOPT_TIMEOUT,           $timeout);
  78     curl_setopt($ch, CURLOPT_USERAGENT,         $userAgent);
  79
  80     if (is_callable($curlWriteFunction)) {
  81         curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
  82     }
  83
  84     // Max download size management
  85     curl_setopt($ch, CURLOPT_BUFFERSIZE,        1024*16);
  86     curl_setopt($ch, CURLOPT_NOPROGRESS,        false);
  87     curl_setopt($ch, CURLOPT_PROGRESSFUNCTION,
  88         function($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes)
  89         {
  90             if (version_compare(phpversion(), '5.5', '<')) {
  91                 // PHP version lower than 5.5
  92                 // Callback has 4 arguments
  93                 $downloaded = $arg1;
  94             } else {
  95                 // Callback has 5 arguments
  96                 $downloaded = $arg2;
  97             }
  98             // Non-zero return stops downloading
  99             return ($downloaded > $maxBytes) ? 1 : 0;
 100         }
 101     );
 102
 103     $response = curl_exec($ch);
 104     $errorNo = curl_errno($ch);
 105     $errorStr = curl_error($ch);
 106     $headSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
 107     curl_close($ch);
 108
 109     if ($response === false) {
 110         if ($errorNo == CURLE_COULDNT_RESOLVE_HOST) {
 111             /*
 112              * Workaround to match fallback method behaviour
 113              * Removing this would require updating
 114              * GetHttpUrlTest::testGetInvalidRemoteUrl()
 115              */
 116             return array(false, false);
 117         }
 118         return array(array(0 => 'curl_exec() error: ' . $errorStr), false);
 119     }
 120
 121     // Formatting output like the fallback method
 122     $rawHeaders = substr($response, 0, $headSize);
 123
 124     // Keep only headers from latest redirection
 125     $rawHeadersArrayRedirs = explode("\r\n\r\n", trim($rawHeaders));
 126     $rawHeadersLastRedir = end($rawHeadersArrayRedirs);
 127
 128     $content = substr($response, $headSize);
 129     $headers = array();
 130     foreach (preg_split('~[\r\n]+~', $rawHeadersLastRedir) as $line) {
 131         if (empty($line) || ctype_space($line)) {
 132             continue;
 133         }
 134         $splitLine = explode(': ', $line, 2);
 135         if (count($splitLine) > 1) {
 136             $key = $splitLine[0];
 137             $value = $splitLine[1];
 138             if (array_key_exists($key, $headers)) {
 139                 if (!is_array($headers[$key])) {
 140                     $headers[$key] = array(0 => $headers[$key]);
 141                 }
 142                 $headers[$key][] = $value;
 143             } else {
 144                 $headers[$key] = $value;
 145             }
 146         } else {
 147             $headers[] = $splitLine[0];
 148         }
 149     }
 150
 151     return array($headers, $content);
 152 }
 153
 154 /**
 155  * GET an HTTP URL to retrieve its content (fallback method)
 156  *
 157  * @param string $cleanUrl       URL to get (http://... valid and in ASCII form)
 158  * @param int    $timeout        network timeout (in seconds)
 159  * @param int    $maxBytes       maximum downloaded bytes
 160  * @param string $userAgent      "User-Agent" header
 161  * @param string $acceptLanguage "Accept-Language" header
 162  * @param int    $maxRedr        maximum amount of redirections followed
 163  *
 164  * @return array HTTP response headers, downloaded content
 165  *
 166  * Output format:
 167  *  [0] = associative array containing HTTP response headers
 168  *  [1] = URL content (downloaded data)
 169  *
 170  * @see http://php.net/manual/en/function.file-get-contents.php
 171  * @see http://php.net/manual/en/function.stream-context-create.php
 172  * @see http://php.net/manual/en/function.get-headers.php
 173  */
 174 function get_http_response_fallback(
 175     $cleanUrl,
 176     $timeout,
 177     $maxBytes,
 178     $userAgent,
 179     $acceptLanguage,
 180     $maxRedr
 181 ) {
 182     $options = array(
 183         'http' => array(
 184             'method' => 'GET',
 185             'timeout' => $timeout,
 186             'user_agent' => $userAgent,
 187             'header' => "Accept: */*\r\n"
 188                 . 'Accept-Language: ' . $acceptLanguage
 189         )
 190     );
 191
 192     stream_context_set_default($options);
 193     list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
 194     if (! $headers || strpos($headers[0], '200 OK') === false) {
 195         $options['http']['request_fulluri'] = true;
 196         stream_context_set_default($options);
 197         list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
 198     }
 199
 200     if (! $headers) {
 201         return array($headers, false);
 202     }
 203
 204     try {
 205         // TODO: catch Exception in calling code (thumbnailer)
 206         $context = stream_context_create($options);
 207         $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
 208     } catch (Exception $exc) {
 209         return array(array(0 => 'HTTP Error'), $exc->getMessage());
 210     }
 211
 212     return array($headers, $content);
 213 }
 214
 215 /**
 216  * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
 217  *
 218  * @param string $url              initial URL to reach.
 219  * @param int    $redirectionLimit max redirection follow.
 220  *
 221  * @return array HTTP headers, or false if it failed.
 222  */
 223 function get_redirected_headers($url, $redirectionLimit = 3)
 224 {
 225     $headers = get_headers($url, 1);
 226     if (!empty($headers['location']) && empty($headers['Location'])) {
 227         $headers['Location'] = $headers['location'];
 228     }
 229
 230     // Headers found, redirection found, and limit not reached.
 231     if ($redirectionLimit-- > 0
 232         && !empty($headers)
 233         && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
 234         && !empty($headers['Location'])) {
 235
 236         $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
 237         if ($redirection != $url) {
 238             $redirection = getAbsoluteUrl($url, $redirection);
 239             return get_redirected_headers($redirection, $redirectionLimit);
 240         }
 241     }
 242
 243     return array($headers, $url);
 244 }
 245
 246 /**
 247  * Get an absolute URL from a complete one, and another absolute/relative URL.
 248  *
 249  * @param string $originalUrl The original complete URL.
 250  * @param string $newUrl      The new one, absolute or relative.
 251  *
 252  * @return string Final URL:
 253  *   - $newUrl if it was already an absolute URL.
 254  *   - if it was relative, absolute URL from $originalUrl path.
 255  */
 256 function getAbsoluteUrl($originalUrl, $newUrl)
 257 {
 258     $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
 259     // Already an absolute URL.
 260     if (!empty($newScheme)) {
 261         return $newUrl;
 262     }
 263
 264     $parts = parse_url($originalUrl);
 265     $final = $parts['scheme'] .'://'. $parts['host'];
 266     $final .= (!empty($parts['port'])) ? $parts['port'] : '';
 267     $final .= '/';
 268     if ($newUrl[0] != '/') {
 269         $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
 270     }
 271     $final .= ltrim($newUrl, '/');
 272     return $final;
 273 }
 274
 275 /**
 276  * Returns the server's base URL: scheme://domain.tld[:port]
 277  *
 278  * @param array $server the $_SERVER array
 279  *
 280  * @return string the server's base URL
 281  *
 282  * @see http://www.ietf.org/rfc/rfc7239.txt
 283  * @see http://www.ietf.org/rfc/rfc6648.txt
 284  * @see http://stackoverflow.com/a/3561399
 285  * @see http://stackoverflow.com/q/452375
 286  */
 287 function server_url($server)
 288 {
 289     $scheme = 'http';
 290     $port = '';
 291
 292     // Shaarli is served behind a proxy
 293     if (isset($server['HTTP_X_FORWARDED_PROTO'])) {
 294         // Keep forwarded scheme
 295         if (strpos($server['HTTP_X_FORWARDED_PROTO'], ',') !== false) {
 296             $schemes = explode(',', $server['HTTP_X_FORWARDED_PROTO']);
 297             $scheme = trim($schemes[0]);
 298         } else {
 299             $scheme = $server['HTTP_X_FORWARDED_PROTO'];
 300         }
 301
 302         if (isset($server['HTTP_X_FORWARDED_PORT'])) {
 303             // Keep forwarded port
 304             if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
 305                 $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
 306                 $port = trim($ports[0]);
 307             } else {
 308                 $port = $server['HTTP_X_FORWARDED_PORT'];
 309             }
 310
 311             // This is a workaround for proxies that don't forward the scheme properly.
 312             // Connecting over port 443 has to be in HTTPS.
 313             // See https://github.com/shaarli/Shaarli/issues/1022
 314             if ($port == '443') {
 315                 $scheme = 'https';
 316             }
 317
 318             if (($scheme == 'http' && $port != '80')
 319                 || ($scheme == 'https' && $port != '443')
 320             ) {
 321                 $port = ':' . $port;
 322             } else {
 323                 $port = '';
 324             }
 325         }
 326
 327         if (isset($server['HTTP_X_FORWARDED_HOST'])) {
 328             // Keep forwarded host
 329             if (strpos($server['HTTP_X_FORWARDED_HOST'], ',') !== false) {
 330                 $hosts = explode(',', $server['HTTP_X_FORWARDED_HOST']);
 331                 $host = trim($hosts[0]);
 332             } else {
 333                 $host = $server['HTTP_X_FORWARDED_HOST'];
 334             }
 335         } else {
 336             $host = $server['SERVER_NAME'];
 337         }
 338
 339         return $scheme.'://'.$host.$port;
 340     }
 341
 342     // SSL detection
 343     if ((! empty($server['HTTPS']) && strtolower($server['HTTPS']) == 'on')
 344         || (isset($server['SERVER_PORT']) && $server['SERVER_PORT'] == '443')) {
 345         $scheme = 'https';
 346     }
 347
 348     // Do not append standard port values
 349     if (($scheme == 'http' && $server['SERVER_PORT'] != '80')
 350         || ($scheme == 'https' && $server['SERVER_PORT'] != '443')) {
 351         $port = ':'.$server['SERVER_PORT'];
 352     }
 353
 354     return $scheme.'://'.$server['SERVER_NAME'].$port;
 355 }
 356
 357 /**
 358  * Returns the absolute URL of the current script, without the query
 359  *
 360  * If the resource is "index.php", then it is removed (for better-looking URLs)
 361  *
 362  * @param array $server the $_SERVER array
 363  *
 364  * @return string the absolute URL of the current script, without the query
 365  */
 366 function index_url($server)
 367 {
 368     $scriptname = $server['SCRIPT_NAME'];
 369     if (endsWith($scriptname, 'index.php')) {
 370         $scriptname = substr($scriptname, 0, -9);
 371     }
 372     return server_url($server) . $scriptname;
 373 }
 374
 375 /**
 376  * Returns the absolute URL of the current script, with the query
 377  *
 378  * If the resource is "index.php", then it is removed (for better-looking URLs)
 379  *
 380  * @param array $server the $_SERVER array
 381  *
 382  * @return string the absolute URL of the current script, with the query
 383  */
 384 function page_url($server)
 385 {
 386     if (! empty($server['QUERY_STRING'])) {
 387         return index_url($server).'?'.$server['QUERY_STRING'];
 388     }
 389     return index_url($server);
 390 }
 391
 392 /**
 393  * Retrieve the initial IP forwarded by the reverse proxy.
 394  *
 395  * Inspired from: https://github.com/zendframework/zend-http/blob/master/src/PhpEnvironment/RemoteAddress.php
 396  *
 397  * @param array $server     $_SERVER array which contains HTTP headers.
 398  * @param array $trustedIps List of trusted IP from the configuration.
 399  *
 400  * @return string|bool The forwarded IP, or false if none could be extracted.
 401  */
 402 function getIpAddressFromProxy($server, $trustedIps)
 403 {
 404     $forwardedIpHeader = 'HTTP_X_FORWARDED_FOR';
 405     if (empty($server[$forwardedIpHeader])) {
 406         return false;
 407     }
 408
 409     $ips = preg_split('/\s*,\s*/', $server[$forwardedIpHeader]);
 410     $ips = array_diff($ips, $trustedIps);
 411     if (empty($ips)) {
 412         return false;
 413     }
 414
 415     return array_pop($ips);
 416 }
 417
 418
 419 /**
 420  * Return an identifier based on the advertised client IP address(es)
 421  *
 422  * This aims at preventing session hijacking from users behind the same proxy
 423  * by relying on HTTP headers.
 424  *
 425  * See:
 426  * - https://secure.php.net/manual/en/reserved.variables.server.php
 427  * - https://stackoverflow.com/questions/3003145/how-to-get-the-client-ip-address-in-php
 428  * - https://stackoverflow.com/questions/12233406/preventing-session-hijacking
 429  * - https://stackoverflow.com/questions/21354859/trusting-x-forwarded-for-to-identify-a-visitor
 430  *
 431  * @param array $server The $_SERVER array
 432  *
 433  * @return string An identifier based on client IP address information
 434  */
 435 function client_ip_id($server)
 436 {
 437     $ip = $server['REMOTE_ADDR'];
 438
 439     if (isset($server['HTTP_X_FORWARDED_FOR'])) {
 440         $ip = $ip . '_' . $server['HTTP_X_FORWARDED_FOR'];
 441     }
 442     if (isset($server['HTTP_CLIENT_IP'])) {
 443         $ip = $ip . '_' . $server['HTTP_CLIENT_IP'];
 444     }
 445     return $ip;
 446 }
 447
 448
 449 /**
 450  * Returns true if Shaarli's currently browsed in HTTPS.
 451  * Supports reverse proxies (if the headers are correctly set).
 452  *
 453  * @param array $server $_SERVER.
 454  *
 455  * @return bool true if HTTPS, false otherwise.
 456  */
 457 function is_https($server)
 458 {
 459
 460     if (isset($server['HTTP_X_FORWARDED_PORT'])) {
 461         // Keep forwarded port
 462         if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
 463             $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
 464             $port = trim($ports[0]);
 465         } else {
 466             $port = $server['HTTP_X_FORWARDED_PORT'];
 467         }
 468
 469         if ($port == '443') {
 470             return true;
 471         }
 472     }
 473
 474     return ! empty($server['HTTPS']);
 475 }