application/HttpUtils.php

   1 <?php
   2 /**
   3  * GET an HTTP URL to retrieve its content
   4  * Uses the cURL library or a fallback method
   5  *
   6  * @param string          $url               URL to get (http://...)
   7  * @param int             $timeout           network timeout (in seconds)
   8  * @param int             $maxBytes          maximum downloaded bytes (default: 4 MiB)
   9  * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
  10  *                                           Can be used to add download conditions on the
  11  *                                           headers (response code, content type, etc.).
  12  *
  13  * @return array HTTP response headers, downloaded content
  14  *
  15  * Output format:
  16  *  [0] = associative array containing HTTP response headers
  17  *  [1] = URL content (downloaded data)
  18  *
  19  * Example:
  20  *  list($headers, $data) = get_http_response('http://sebauvage.net/');
  21  *  if (strpos($headers[0], '200 OK') !== false) {
  22  *      echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
  23  *  } else {
  24  *      echo 'There was an error: '.htmlspecialchars($headers[0]);
  25  *  }
  26  *
  27  * @see https://secure.php.net/manual/en/ref.curl.php
  28  * @see https://secure.php.net/manual/en/functions.anonymous.php
  29  * @see https://secure.php.net/manual/en/function.preg-split.php
  30  * @see https://secure.php.net/manual/en/function.explode.php
  31  * @see http://stackoverflow.com/q/17641073
  32  * @see http://stackoverflow.com/q/9183178
  33  * @see http://stackoverflow.com/q/1462720
  34  */
  35 function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null)
  36 {
  37     $urlObj = new Url($url);
  38     $cleanUrl = $urlObj->idnToAscii();
  39
  40     if (!filter_var($cleanUrl, FILTER_VALIDATE_URL) || !$urlObj->isHttp()) {
  41         return array(array(0 => 'Invalid HTTP Url'), false);
  42     }
  43
  44     $userAgent =
  45         'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
  46         . ' Gecko/20100101 Firefox/45.0';
  47     $acceptLanguage =
  48         substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3';
  49     $maxRedirs = 3;
  50
  51     if (!function_exists('curl_init')) {
  52         return get_http_response_fallback(
  53             $cleanUrl,
  54             $timeout,
  55             $maxBytes,
  56             $userAgent,
  57             $acceptLanguage,
  58             $maxRedirs
  59         );
  60     }
  61
  62     $ch = curl_init($cleanUrl);
  63     if ($ch === false) {
  64         return array(array(0 => 'curl_init() error'), false);
  65     }
  66
  67     // General cURL settings
  68     curl_setopt($ch, CURLOPT_AUTOREFERER, true);
  69     curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
  70     curl_setopt($ch, CURLOPT_HEADER, true);
  71     curl_setopt(
  72         $ch,
  73         CURLOPT_HTTPHEADER,
  74         array('Accept-Language: ' . $acceptLanguage)
  75     );
  76     curl_setopt($ch, CURLOPT_MAXREDIRS, $maxRedirs);
  77     curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  78     curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
  79     curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
  80
  81     if (is_callable($curlWriteFunction)) {
  82         curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
  83     }
  84
  85     // Max download size management
  86     curl_setopt($ch, CURLOPT_BUFFERSIZE, 1024*16);
  87     curl_setopt($ch, CURLOPT_NOPROGRESS, false);
  88     curl_setopt(
  89         $ch,
  90         CURLOPT_PROGRESSFUNCTION,
  91         function ($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes) {
  92             if (version_compare(phpversion(), '5.5', '<')) {
  93                 // PHP version lower than 5.5
  94                 // Callback has 4 arguments
  95                 $downloaded = $arg1;
  96             } else {
  97                 // Callback has 5 arguments
  98                 $downloaded = $arg2;
  99             }
 100             // Non-zero return stops downloading
 101             return ($downloaded > $maxBytes) ? 1 : 0;
 102         }
 103     );
 104
 105     $response = curl_exec($ch);
 106     $errorNo = curl_errno($ch);
 107     $errorStr = curl_error($ch);
 108     $headSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
 109     curl_close($ch);
 110
 111     if ($response === false) {
 112         if ($errorNo == CURLE_COULDNT_RESOLVE_HOST) {
 113             /*
 114              * Workaround to match fallback method behaviour
 115              * Removing this would require updating
 116              * GetHttpUrlTest::testGetInvalidRemoteUrl()
 117              */
 118             return array(false, false);
 119         }
 120         return array(array(0 => 'curl_exec() error: ' . $errorStr), false);
 121     }
 122
 123     // Formatting output like the fallback method
 124     $rawHeaders = substr($response, 0, $headSize);
 125
 126     // Keep only headers from latest redirection
 127     $rawHeadersArrayRedirs = explode("\r\n\r\n", trim($rawHeaders));
 128     $rawHeadersLastRedir = end($rawHeadersArrayRedirs);
 129
 130     $content = substr($response, $headSize);
 131     $headers = array();
 132     foreach (preg_split('~[\r\n]+~', $rawHeadersLastRedir) as $line) {
 133         if (empty($line) || ctype_space($line)) {
 134             continue;
 135         }
 136         $splitLine = explode(': ', $line, 2);
 137         if (count($splitLine) > 1) {
 138             $key = $splitLine[0];
 139             $value = $splitLine[1];
 140             if (array_key_exists($key, $headers)) {
 141                 if (!is_array($headers[$key])) {
 142                     $headers[$key] = array(0 => $headers[$key]);
 143                 }
 144                 $headers[$key][] = $value;
 145             } else {
 146                 $headers[$key] = $value;
 147             }
 148         } else {
 149             $headers[] = $splitLine[0];
 150         }
 151     }
 152
 153     return array($headers, $content);
 154 }
 155
 156 /**
 157  * GET an HTTP URL to retrieve its content (fallback method)
 158  *
 159  * @param string $cleanUrl       URL to get (http://... valid and in ASCII form)
 160  * @param int    $timeout        network timeout (in seconds)
 161  * @param int    $maxBytes       maximum downloaded bytes
 162  * @param string $userAgent      "User-Agent" header
 163  * @param string $acceptLanguage "Accept-Language" header
 164  * @param int    $maxRedr        maximum amount of redirections followed
 165  *
 166  * @return array HTTP response headers, downloaded content
 167  *
 168  * Output format:
 169  *  [0] = associative array containing HTTP response headers
 170  *  [1] = URL content (downloaded data)
 171  *
 172  * @see http://php.net/manual/en/function.file-get-contents.php
 173  * @see http://php.net/manual/en/function.stream-context-create.php
 174  * @see http://php.net/manual/en/function.get-headers.php
 175  */
 176 function get_http_response_fallback(
 177     $cleanUrl,
 178     $timeout,
 179     $maxBytes,
 180     $userAgent,
 181     $acceptLanguage,
 182     $maxRedr
 183 ) {
 184     $options = array(
 185         'http' => array(
 186             'method' => 'GET',
 187             'timeout' => $timeout,
 188             'user_agent' => $userAgent,
 189             'header' => "Accept: */*\r\n"
 190                 . 'Accept-Language: ' . $acceptLanguage
 191         )
 192     );
 193
 194     stream_context_set_default($options);
 195     list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
 196     if (! $headers || strpos($headers[0], '200 OK') === false) {
 197         $options['http']['request_fulluri'] = true;
 198         stream_context_set_default($options);
 199         list($headers, $finalUrl) = get_redirected_headers($cleanUrl, $maxRedr);
 200     }
 201
 202     if (! $headers) {
 203         return array($headers, false);
 204     }
 205
 206     try {
 207         // TODO: catch Exception in calling code (thumbnailer)
 208         $context = stream_context_create($options);
 209         $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
 210     } catch (Exception $exc) {
 211         return array(array(0 => 'HTTP Error'), $exc->getMessage());
 212     }
 213
 214     return array($headers, $content);
 215 }
 216
 217 /**
 218  * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
 219  *
 220  * @param string $url              initial URL to reach.
 221  * @param int    $redirectionLimit max redirection follow.
 222  *
 223  * @return array HTTP headers, or false if it failed.
 224  */
 225 function get_redirected_headers($url, $redirectionLimit = 3)
 226 {
 227     $headers = get_headers($url, 1);
 228     if (!empty($headers['location']) && empty($headers['Location'])) {
 229         $headers['Location'] = $headers['location'];
 230     }
 231
 232     // Headers found, redirection found, and limit not reached.
 233     if ($redirectionLimit-- > 0
 234         && !empty($headers)
 235         && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
 236         && !empty($headers['Location'])) {
 237         $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
 238         if ($redirection != $url) {
 239             $redirection = getAbsoluteUrl($url, $redirection);
 240             return get_redirected_headers($redirection, $redirectionLimit);
 241         }
 242     }
 243
 244     return array($headers, $url);
 245 }
 246
 247 /**
 248  * Get an absolute URL from a complete one, and another absolute/relative URL.
 249  *
 250  * @param string $originalUrl The original complete URL.
 251  * @param string $newUrl      The new one, absolute or relative.
 252  *
 253  * @return string Final URL:
 254  *   - $newUrl if it was already an absolute URL.
 255  *   - if it was relative, absolute URL from $originalUrl path.
 256  */
 257 function getAbsoluteUrl($originalUrl, $newUrl)
 258 {
 259     $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
 260     // Already an absolute URL.
 261     if (!empty($newScheme)) {
 262         return $newUrl;
 263     }
 264
 265     $parts = parse_url($originalUrl);
 266     $final = $parts['scheme'] .'://'. $parts['host'];
 267     $final .= (!empty($parts['port'])) ? $parts['port'] : '';
 268     $final .= '/';
 269     if ($newUrl[0] != '/') {
 270         $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
 271     }
 272     $final .= ltrim($newUrl, '/');
 273     return $final;
 274 }
 275
 276 /**
 277  * Returns the server's base URL: scheme://domain.tld[:port]
 278  *
 279  * @param array $server the $_SERVER array
 280  *
 281  * @return string the server's base URL
 282  *
 283  * @see http://www.ietf.org/rfc/rfc7239.txt
 284  * @see http://www.ietf.org/rfc/rfc6648.txt
 285  * @see http://stackoverflow.com/a/3561399
 286  * @see http://stackoverflow.com/q/452375
 287  */
 288 function server_url($server)
 289 {
 290     $scheme = 'http';
 291     $port = '';
 292
 293     // Shaarli is served behind a proxy
 294     if (isset($server['HTTP_X_FORWARDED_PROTO'])) {
 295         // Keep forwarded scheme
 296         if (strpos($server['HTTP_X_FORWARDED_PROTO'], ',') !== false) {
 297             $schemes = explode(',', $server['HTTP_X_FORWARDED_PROTO']);
 298             $scheme = trim($schemes[0]);
 299         } else {
 300             $scheme = $server['HTTP_X_FORWARDED_PROTO'];
 301         }
 302
 303         if (isset($server['HTTP_X_FORWARDED_PORT'])) {
 304             // Keep forwarded port
 305             if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
 306                 $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
 307                 $port = trim($ports[0]);
 308             } else {
 309                 $port = $server['HTTP_X_FORWARDED_PORT'];
 310             }
 311
 312             // This is a workaround for proxies that don't forward the scheme properly.
 313             // Connecting over port 443 has to be in HTTPS.
 314             // See https://github.com/shaarli/Shaarli/issues/1022
 315             if ($port == '443') {
 316                 $scheme = 'https';
 317             }
 318
 319             if (($scheme == 'http' && $port != '80')
 320                 || ($scheme == 'https' && $port != '443')
 321             ) {
 322                 $port = ':' . $port;
 323             } else {
 324                 $port = '';
 325             }
 326         }
 327
 328         if (isset($server['HTTP_X_FORWARDED_HOST'])) {
 329             // Keep forwarded host
 330             if (strpos($server['HTTP_X_FORWARDED_HOST'], ',') !== false) {
 331                 $hosts = explode(',', $server['HTTP_X_FORWARDED_HOST']);
 332                 $host = trim($hosts[0]);
 333             } else {
 334                 $host = $server['HTTP_X_FORWARDED_HOST'];
 335             }
 336         } else {
 337             $host = $server['SERVER_NAME'];
 338         }
 339
 340         return $scheme.'://'.$host.$port;
 341     }
 342
 343     // SSL detection
 344     if ((! empty($server['HTTPS']) && strtolower($server['HTTPS']) == 'on')
 345         || (isset($server['SERVER_PORT']) && $server['SERVER_PORT'] == '443')) {
 346         $scheme = 'https';
 347     }
 348
 349     // Do not append standard port values
 350     if (($scheme == 'http' && $server['SERVER_PORT'] != '80')
 351         || ($scheme == 'https' && $server['SERVER_PORT'] != '443')) {
 352         $port = ':'.$server['SERVER_PORT'];
 353     }
 354
 355     return $scheme.'://'.$server['SERVER_NAME'].$port;
 356 }
 357
 358 /**
 359  * Returns the absolute URL of the current script, without the query
 360  *
 361  * If the resource is "index.php", then it is removed (for better-looking URLs)
 362  *
 363  * @param array $server the $_SERVER array
 364  *
 365  * @return string the absolute URL of the current script, without the query
 366  */
 367 function index_url($server)
 368 {
 369     $scriptname = $server['SCRIPT_NAME'];
 370     if (endsWith($scriptname, 'index.php')) {
 371         $scriptname = substr($scriptname, 0, -9);
 372     }
 373     return server_url($server) . $scriptname;
 374 }
 375
 376 /**
 377  * Returns the absolute URL of the current script, with the query
 378  *
 379  * If the resource is "index.php", then it is removed (for better-looking URLs)
 380  *
 381  * @param array $server the $_SERVER array
 382  *
 383  * @return string the absolute URL of the current script, with the query
 384  */
 385 function page_url($server)
 386 {
 387     if (! empty($server['QUERY_STRING'])) {
 388         return index_url($server).'?'.$server['QUERY_STRING'];
 389     }
 390     return index_url($server);
 391 }
 392
 393 /**
 394  * Retrieve the initial IP forwarded by the reverse proxy.
 395  *
 396  * Inspired from: https://github.com/zendframework/zend-http/blob/master/src/PhpEnvironment/RemoteAddress.php
 397  *
 398  * @param array $server     $_SERVER array which contains HTTP headers.
 399  * @param array $trustedIps List of trusted IP from the configuration.
 400  *
 401  * @return string|bool The forwarded IP, or false if none could be extracted.
 402  */
 403 function getIpAddressFromProxy($server, $trustedIps)
 404 {
 405     $forwardedIpHeader = 'HTTP_X_FORWARDED_FOR';
 406     if (empty($server[$forwardedIpHeader])) {
 407         return false;
 408     }
 409
 410     $ips = preg_split('/\s*,\s*/', $server[$forwardedIpHeader]);
 411     $ips = array_diff($ips, $trustedIps);
 412     if (empty($ips)) {
 413         return false;
 414     }
 415
 416     return array_pop($ips);
 417 }
 418
 419
 420 /**
 421  * Return an identifier based on the advertised client IP address(es)
 422  *
 423  * This aims at preventing session hijacking from users behind the same proxy
 424  * by relying on HTTP headers.
 425  *
 426  * See:
 427  * - https://secure.php.net/manual/en/reserved.variables.server.php
 428  * - https://stackoverflow.com/questions/3003145/how-to-get-the-client-ip-address-in-php
 429  * - https://stackoverflow.com/questions/12233406/preventing-session-hijacking
 430  * - https://stackoverflow.com/questions/21354859/trusting-x-forwarded-for-to-identify-a-visitor
 431  *
 432  * @param array $server The $_SERVER array
 433  *
 434  * @return string An identifier based on client IP address information
 435  */
 436 function client_ip_id($server)
 437 {
 438     $ip = $server['REMOTE_ADDR'];
 439
 440     if (isset($server['HTTP_X_FORWARDED_FOR'])) {
 441         $ip = $ip . '_' . $server['HTTP_X_FORWARDED_FOR'];
 442     }
 443     if (isset($server['HTTP_CLIENT_IP'])) {
 444         $ip = $ip . '_' . $server['HTTP_CLIENT_IP'];
 445     }
 446     return $ip;
 447 }
 448
 449
 450 /**
 451  * Returns true if Shaarli's currently browsed in HTTPS.
 452  * Supports reverse proxies (if the headers are correctly set).
 453  *
 454  * @param array $server $_SERVER.
 455  *
 456  * @return bool true if HTTPS, false otherwise.
 457  */
 458 function is_https($server)
 459 {
 460
 461     if (isset($server['HTTP_X_FORWARDED_PORT'])) {
 462         // Keep forwarded port
 463         if (strpos($server['HTTP_X_FORWARDED_PORT'], ',') !== false) {
 464             $ports = explode(',', $server['HTTP_X_FORWARDED_PORT']);
 465             $port = trim($ports[0]);
 466         } else {
 467             $port = $server['HTTP_X_FORWARDED_PORT'];
 468         }
 469
 470         if ($port == '443') {
 471             return true;
 472         }
 473     }
 474
 475     return ! empty($server['HTTPS']);
 476 }