From ce7b0b6480aa854ee6893f5c889277b0e3b13efc Mon Sep 17 00:00:00 2001 From: ArthurHoaro Date: Wed, 6 Apr 2016 22:00:52 +0200 Subject: Fixes #531 - Title retrieving is failing with multiple use case see https://github.com/shaarli/Shaarli/issues/531 for details --- application/HttpUtils.php | 60 +++++++++++++++++++++++++++++++++++++++-------- application/LinkUtils.php | 6 ++--- application/Url.php | 42 +++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 13 deletions(-) (limited to 'application') diff --git a/application/HttpUtils.php b/application/HttpUtils.php index af7cb371..0e1ce879 100644 --- a/application/HttpUtils.php +++ b/application/HttpUtils.php @@ -27,7 +27,9 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) { $urlObj = new Url($url); - if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) { + $cleanUrl = $urlObj->indToAscii(); + + if (! filter_var($cleanUrl, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) { return array(array(0 => 'Invalid HTTP Url'), false); } @@ -35,22 +37,27 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) 'http' => array( 'method' => 'GET', 'timeout' => $timeout, - 'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)' - .' Gecko/20100101 Firefox/23.0', - 'request_fulluri' => true, + 'user_agent' => 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)' + .' Gecko/20100101 Firefox/45.0', + 'accept_language' => substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3', ) ); - $context = stream_context_create($options); stream_context_set_default($options); + list($headers, $finalUrl) = get_redirected_headers($cleanUrl); + if (! $headers || strpos($headers[0], '200 OK') === false) { + $options['http']['request_fulluri'] = true; + stream_context_set_default($options); + list($headers, $finalUrl) = get_redirected_headers($cleanUrl); + } - list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup()); if (! $headers || strpos($headers[0], '200 OK') === false) { return array($headers, false); } try { // TODO: catch Exception in calling code (thumbnailer) + $context = stream_context_create($options); $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes); } catch (Exception $exc) { return array(array(0 => 'HTTP Error'), $exc->getMessage()); @@ -60,16 +67,19 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) } /** - * Retrieve HTTP headers, following n redirections (temporary and permanent). + * Retrieve HTTP headers, following n redirections (temporary and permanent ones). * - * @param string $url initial URL to reach. - * @param int $redirectionLimit max redirection follow.. + * @param string $url initial URL to reach. + * @param int $redirectionLimit max redirection follow.. * - * @return array + * @return array HTTP headers, or false if it failed. */ function get_redirected_headers($url, $redirectionLimit = 3) { $headers = get_headers($url, 1); + if (!empty($headers['location']) && empty($headers['Location'])) { + $headers['Location'] = $headers['location']; + } // Headers found, redirection found, and limit not reached. if ($redirectionLimit-- > 0 @@ -79,6 +89,7 @@ function get_redirected_headers($url, $redirectionLimit = 3) $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location']; if ($redirection != $url) { + $redirection = getAbsoluteUrl($url, $redirection); return get_redirected_headers($redirection, $redirectionLimit); } } @@ -86,6 +97,35 @@ function get_redirected_headers($url, $redirectionLimit = 3) return array($headers, $url); } +/** + * Get an absolute URL from a complete one, and another absolute/relative URL. + * + * @param string $originalUrl The original complete URL. + * @param string $newUrl The new one, absolute or relative. + * + * @return string Final URL: + * - $newUrl if it was already an absolute URL. + * - if it was relative, absolute URL from $originalUrl path. + */ +function getAbsoluteUrl($originalUrl, $newUrl) +{ + $newScheme = parse_url($newUrl, PHP_URL_SCHEME); + // Already an absolute URL. + if (!empty($newScheme)) { + return $newUrl; + } + + $parts = parse_url($originalUrl); + $final = $parts['scheme'] .'://'. $parts['host']; + $final .= (!empty($parts['port'])) ? $parts['port'] : ''; + $final .= '/'; + if ($newUrl[0] != '/') { + $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/')); + } + $final .= ltrim($newUrl, '/'); + return $final; +} + /** * Returns the server's base URL: scheme://domain.tld[:port] * diff --git a/application/LinkUtils.php b/application/LinkUtils.php index d8dc8b5e..2df76ba8 100644 --- a/application/LinkUtils.php +++ b/application/LinkUtils.php @@ -9,8 +9,8 @@ */ function html_extract_title($html) { - if (preg_match('!(.*?)!is', $html, $matches)) { - return trim(str_replace("\n", ' ', $matches[1])); + if (preg_match('!(.*?)!is', $html, $matches)) { + return trim(str_replace("\n", '', $matches[1])); } return false; } @@ -70,7 +70,7 @@ function headers_extract_charset($headers) function html_extract_charset($html) { // Get encoding specified in HTML header. - preg_match('#/]+)"? */?>#Usi', $html, $enc); + preg_match('#/]+)["\']? */?>#Usi', $html, $enc); if (!empty($enc[1])) { return strtolower($enc[1]); } diff --git a/application/Url.php b/application/Url.php index af38c4d9..61a30a78 100644 --- a/application/Url.php +++ b/application/Url.php @@ -62,7 +62,21 @@ function add_trailing_slash($url) { return $url . (!endsWith($url, '/') ? '/' : ''); } +/** + * Converts an URL with an IDN host to a ASCII one. + * + * @param string $url Input URL. + * + * @return string converted URL. + */ +function url_with_idn_to_ascii($url) +{ + $parts = parse_url($url); + $parts['host'] = idn_to_ascii($parts['host']); + $httpUrl = new \http\Url($parts); + return $httpUrl->toString(); +} /** * URL representation and cleanup utilities * @@ -220,6 +234,22 @@ class Url return $this->toString(); } + /** + * Converts an URL with an International Domain Name host to a ASCII one. + * This requires PHP-intl. If it's not available, just returns this->cleanup(). + * + * @return string converted cleaned up URL. + */ + public function indToAscii() + { + $out = $this->cleanup(); + if (! function_exists('idn_to_ascii') || ! isset($this->parts['host'])) { + return $out; + } + $asciiHost = idn_to_ascii($this->parts['host']); + return str_replace($this->parts['host'], $asciiHost, $out); + } + /** * Get URL scheme. * @@ -232,6 +262,18 @@ class Url return $this->parts['scheme']; } + /** + * Get URL host. + * + * @return string the URL host or false if none is provided. + */ + public function getHost() { + if (empty($this->parts['host'])) { + return false; + } + return $this->parts['host']; + } + /** * Test if the Url is an HTTP one. * -- cgit v1.2.3