From 1557cefbd76257ceb830f65806831b490faf0acc Mon Sep 17 00:00:00 2001 From: ArthurHoaro Date: Mon, 4 Jan 2016 10:45:54 +0100 Subject: Fixes #410 - Retrieve title fails in multiple cases * `get_http_url()` renamed to `get_http_response()`. * Use the same HTTP context to retrieve response headers and content. * Follow HTTP 301 and 302 redirections to retrieve the title (default max 3 redirections). * Add `LinkUtils` to extract titles and charset. * Try to retrieve charset from HTTP headers first (new), then HTML content. * Use mb_string to re-encode title if necessary. --- application/ApplicationUtils.php | 2 +- application/HttpUtils.php | 49 +++++++++++++++++++++---- application/LinkUtils.php | 79 ++++++++++++++++++++++++++++++++++++++++ application/Url.php | 11 +++++- 4 files changed, 132 insertions(+), 9 deletions(-) mode change 100644 => 100755 application/HttpUtils.php create mode 100755 application/LinkUtils.php mode change 100644 => 100755 application/Url.php (limited to 'application') diff --git a/application/ApplicationUtils.php b/application/ApplicationUtils.php index 274331e1..978fc9da 100644 --- a/application/ApplicationUtils.php +++ b/application/ApplicationUtils.php @@ -19,7 +19,7 @@ class ApplicationUtils */ public static function getLatestGitVersionCode($url, $timeout=2) { - list($headers, $data) = get_http_url($url, $timeout); + list($headers, $data) = get_http_response($url, $timeout); if (strpos($headers[0], '200 OK') === false) { error_log('Failed to retrieve ' . $url); diff --git a/application/HttpUtils.php b/application/HttpUtils.php old mode 100644 new mode 100755 index 499220c5..e2c1cb47 --- a/application/HttpUtils.php +++ b/application/HttpUtils.php @@ -13,7 +13,7 @@ * [1] = URL content (downloaded data) * * Example: - * list($headers, $data) = get_http_url('http://sebauvage.net/'); + * list($headers, $data) = get_http_response('http://sebauvage.net/'); * if (strpos($headers[0], '200 OK') !== false) { * echo 'Data type: '.htmlspecialchars($headers['Content-Type']); * } else { @@ -24,31 +24,66 @@ * @see http://php.net/manual/en/function.stream-context-create.php * @see http://php.net/manual/en/function.get-headers.php */ -function get_http_url($url, $timeout = 30, $maxBytes = 4194304) +function get_http_response($url, $timeout = 30, $maxBytes = 4194304) { + $urlObj = new Url($url); + if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) { + return array(array(0 => 'Invalid HTTP Url'), false); + } + $options = array( 'http' => array( 'method' => 'GET', 'timeout' => $timeout, 'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)' - .' Gecko/20100101 Firefox/23.0' + .' Gecko/20100101 Firefox/23.0', + 'request_fulluri' => true, ) ); $context = stream_context_create($options); + stream_context_set_default($options); + + list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup()); + if (! $headers || strpos($headers[0], '200 OK') === false) { + return array($headers, false); + } try { // TODO: catch Exception in calling code (thumbnailer) - $content = file_get_contents($url, false, $context, -1, $maxBytes); + $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes); } catch (Exception $exc) { return array(array(0 => 'HTTP Error'), $exc->getMessage()); } - if (!$content) { - return array(array(0 => 'HTTP Error'), ''); + return array($headers, $content); +} + +/** + * Retrieve HTTP headers, following n redirections (temporary and permanent). + * + * @param string $url initial URL to reach. + * @param int $redirectionLimit max redirection follow.. + * + * @return array + */ +function get_redirected_headers($url, $redirectionLimit = 3) +{ + $headers = get_headers($url, 1); + + // Headers found, redirection found, and limit not reached. + if ($redirectionLimit-- > 0 + && !empty($headers) + && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false) + && !empty($headers['Location'])) { + + $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location']; + if ($redirection != $url) { + return get_redirected_headers($redirection, $redirectionLimit); + } } - return array(get_headers($url, 1), $content); + return array($headers, $url); } /** diff --git a/application/LinkUtils.php b/application/LinkUtils.php new file mode 100755 index 00000000..26dd6b67 --- /dev/null +++ b/application/LinkUtils.php @@ -0,0 +1,79 @@ +(.*)!is', $html, $matches)) { + return trim(str_replace("\n", ' ', $matches[1])); + } + return false; +} + +/** + * Determine charset from downloaded page. + * Priority: + * 1. HTTP headers (Content type). + * 2. HTML content page (tag ). + * 3. Use a default charset (default: UTF-8). + * + * @param array $headers HTTP headers array. + * @param string $htmlContent HTML content where to look for charset. + * @param string $defaultCharset Default charset to apply if other methods failed. + * + * @return string Determined charset. + */ +function get_charset($headers, $htmlContent, $defaultCharset = 'utf-8') +{ + if ($charset = headers_extract_charset($headers)) { + return $charset; + } + + if ($charset = html_extract_charset($htmlContent)) { + return $charset; + } + + return $defaultCharset; +} + +/** + * Extract charset from HTTP headers if it's defined. + * + * @param array $headers HTTP headers array. + * + * @return bool|string Charset string if found (lowercase), false otherwise. + */ +function headers_extract_charset($headers) +{ + if (! empty($headers['Content-Type']) && strpos($headers['Content-Type'], 'charset=') !== false) { + preg_match('/charset="?([^; ]+)/i', $headers['Content-Type'], $match); + if (! empty($match[1])) { + return strtolower(trim($match[1])); + } + } + + return false; +} + +/** + * Extract charset HTML content (tag ). + * + * @param string $html HTML content where to look for charset. + * + * @return bool|string Charset string if found, false otherwise. + */ +function html_extract_charset($html) +{ + // Get encoding specified in HTML header. + preg_match('#/]+)"? */?>#Usi', $html, $enc); + if (!empty($enc[1])) { + return strtolower($enc[1]); + } + + return false; +} diff --git a/application/Url.php b/application/Url.php old mode 100644 new mode 100755 index d80c9c58..a4ac2e73 --- a/application/Url.php +++ b/application/Url.php @@ -118,7 +118,7 @@ class Url */ public function __construct($url) { - $this->parts = parse_url($url); + $this->parts = parse_url(trim($url)); if (!empty($url) && empty($this->parts['scheme'])) { $this->parts['scheme'] = 'http'; @@ -201,4 +201,13 @@ class Url } return $this->parts['scheme']; } + + /** + * Test if the Url is an HTTP one. + * + * @return true is HTTP, false otherwise. + */ + public function isHttp() { + return strpos(strtolower($this->parts['scheme']), 'http') !== false; + } } -- cgit v1.2.3