From 1557cefbd76257ceb830f65806831b490faf0acc Mon Sep 17 00:00:00 2001 From: ArthurHoaro Date: Mon, 4 Jan 2016 10:45:54 +0100 Subject: Fixes #410 - Retrieve title fails in multiple cases * `get_http_url()` renamed to `get_http_response()`. * Use the same HTTP context to retrieve response headers and content. * Follow HTTP 301 and 302 redirections to retrieve the title (default max 3 redirections). * Add `LinkUtils` to extract titles and charset. * Try to retrieve charset from HTTP headers first (new), then HTML content. * Use mb_string to re-encode title if necessary. --- application/ApplicationUtils.php | 2 +- application/HttpUtils.php | 49 ++++++++++++++++++--- application/LinkUtils.php | 79 ++++++++++++++++++++++++++++++++++ application/Url.php | 11 ++++- index.php | 87 ++++++++++++++++---------------------- tests/HttpUtils/GetHttpUrlTest.php | 26 ++++++------ tests/LinkUtilsTest.php | 85 +++++++++++++++++++++++++++++++++++++ tests/Url/UrlTest.php | 18 ++++++++ 8 files changed, 285 insertions(+), 72 deletions(-) mode change 100644 => 100755 application/HttpUtils.php create mode 100755 application/LinkUtils.php mode change 100644 => 100755 application/Url.php create mode 100644 tests/LinkUtilsTest.php diff --git a/application/ApplicationUtils.php b/application/ApplicationUtils.php index 274331e1..978fc9da 100644 --- a/application/ApplicationUtils.php +++ b/application/ApplicationUtils.php @@ -19,7 +19,7 @@ class ApplicationUtils */ public static function getLatestGitVersionCode($url, $timeout=2) { - list($headers, $data) = get_http_url($url, $timeout); + list($headers, $data) = get_http_response($url, $timeout); if (strpos($headers[0], '200 OK') === false) { error_log('Failed to retrieve ' . $url); diff --git a/application/HttpUtils.php b/application/HttpUtils.php old mode 100644 new mode 100755 index 499220c5..e2c1cb47 --- a/application/HttpUtils.php +++ b/application/HttpUtils.php @@ -13,7 +13,7 @@ * [1] = URL content (downloaded data) * * Example: - * list($headers, $data) = get_http_url('http://sebauvage.net/'); + * list($headers, $data) = get_http_response('http://sebauvage.net/'); * if (strpos($headers[0], '200 OK') !== false) { * echo 'Data type: '.htmlspecialchars($headers['Content-Type']); * } else { @@ -24,31 +24,66 @@ * @see http://php.net/manual/en/function.stream-context-create.php * @see http://php.net/manual/en/function.get-headers.php */ -function get_http_url($url, $timeout = 30, $maxBytes = 4194304) +function get_http_response($url, $timeout = 30, $maxBytes = 4194304) { + $urlObj = new Url($url); + if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) { + return array(array(0 => 'Invalid HTTP Url'), false); + } + $options = array( 'http' => array( 'method' => 'GET', 'timeout' => $timeout, 'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)' - .' Gecko/20100101 Firefox/23.0' + .' Gecko/20100101 Firefox/23.0', + 'request_fulluri' => true, ) ); $context = stream_context_create($options); + stream_context_set_default($options); + + list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup()); + if (! $headers || strpos($headers[0], '200 OK') === false) { + return array($headers, false); + } try { // TODO: catch Exception in calling code (thumbnailer) - $content = file_get_contents($url, false, $context, -1, $maxBytes); + $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes); } catch (Exception $exc) { return array(array(0 => 'HTTP Error'), $exc->getMessage()); } - if (!$content) { - return array(array(0 => 'HTTP Error'), ''); + return array($headers, $content); +} + +/** + * Retrieve HTTP headers, following n redirections (temporary and permanent). + * + * @param string $url initial URL to reach. + * @param int $redirectionLimit max redirection follow.. + * + * @return array + */ +function get_redirected_headers($url, $redirectionLimit = 3) +{ + $headers = get_headers($url, 1); + + // Headers found, redirection found, and limit not reached. + if ($redirectionLimit-- > 0 + && !empty($headers) + && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false) + && !empty($headers['Location'])) { + + $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location']; + if ($redirection != $url) { + return get_redirected_headers($redirection, $redirectionLimit); + } } - return array(get_headers($url, 1), $content); + return array($headers, $url); } /** diff --git a/application/LinkUtils.php b/application/LinkUtils.php new file mode 100755 index 00000000..26dd6b67 --- /dev/null +++ b/application/LinkUtils.php @@ -0,0 +1,79 @@ +(.*)!is', $html, $matches)) { + return trim(str_replace("\n", ' ', $matches[1])); + } + return false; +} + +/** + * Determine charset from downloaded page. + * Priority: + * 1. HTTP headers (Content type). + * 2. HTML content page (tag ). + * 3. Use a default charset (default: UTF-8). + * + * @param array $headers HTTP headers array. + * @param string $htmlContent HTML content where to look for charset. + * @param string $defaultCharset Default charset to apply if other methods failed. + * + * @return string Determined charset. + */ +function get_charset($headers, $htmlContent, $defaultCharset = 'utf-8') +{ + if ($charset = headers_extract_charset($headers)) { + return $charset; + } + + if ($charset = html_extract_charset($htmlContent)) { + return $charset; + } + + return $defaultCharset; +} + +/** + * Extract charset from HTTP headers if it's defined. + * + * @param array $headers HTTP headers array. + * + * @return bool|string Charset string if found (lowercase), false otherwise. + */ +function headers_extract_charset($headers) +{ + if (! empty($headers['Content-Type']) && strpos($headers['Content-Type'], 'charset=') !== false) { + preg_match('/charset="?([^; ]+)/i', $headers['Content-Type'], $match); + if (! empty($match[1])) { + return strtolower(trim($match[1])); + } + } + + return false; +} + +/** + * Extract charset HTML content (tag ). + * + * @param string $html HTML content where to look for charset. + * + * @return bool|string Charset string if found, false otherwise. + */ +function html_extract_charset($html) +{ + // Get encoding specified in HTML header. + preg_match('#/]+)"? */?>#Usi', $html, $enc); + if (!empty($enc[1])) { + return strtolower($enc[1]); + } + + return false; +} diff --git a/application/Url.php b/application/Url.php old mode 100644 new mode 100755 index d80c9c58..a4ac2e73 --- a/application/Url.php +++ b/application/Url.php @@ -118,7 +118,7 @@ class Url */ public function __construct($url) { - $this->parts = parse_url($url); + $this->parts = parse_url(trim($url)); if (!empty($url) && empty($this->parts['scheme'])) { $this->parts['scheme'] = 'http'; @@ -201,4 +201,13 @@ class Url } return $this->parts['scheme']; } + + /** + * Test if the Url is an HTTP one. + * + * @return true is HTTP, false otherwise. + */ + public function isHttp() { + return strpos(strtolower($this->parts['scheme']), 'http') !== false; + } } diff --git a/index.php b/index.php index cd83600b..600b2f55 100644 --- a/index.php +++ b/index.php @@ -152,6 +152,7 @@ require_once 'application/FileUtils.php'; require_once 'application/HttpUtils.php'; require_once 'application/LinkDB.php'; require_once 'application/LinkFilter.php'; +require_once 'application/LinkUtils.php'; require_once 'application/TimeZone.php'; require_once 'application/Url.php'; require_once 'application/Utils.php'; @@ -578,13 +579,6 @@ function linkdate2iso8601($linkdate) return date('c',linkdate2timestamp($linkdate)); // 'c' is for ISO 8601 date format. } -// Extract title from an HTML document. -// (Returns an empty string if not found.) -function html_extract_title($html) -{ - return preg_match('!(.*?)!is', $html, $matches) ? trim(str_replace("\n",' ', $matches[1])) : '' ; -} - // ------------------------------------------------------------------------------------------ // Token management for XSRF protection // Token should be used in any form which acts on data (create,update,delete,import...). @@ -1642,7 +1636,7 @@ function renderPage() // -------- User want to post a new link: Display link edit form. if (isset($_GET['post'])) { - $url = cleanup_url($_GET['post']); + $url = cleanup_url(escape($_GET['post'])); $link_is_new = false; // Check if URL is not already in database (in this case, we will edit the existing link) @@ -1660,35 +1654,24 @@ function renderPage() // If this is an HTTP(S) link, we try go get the page to extract the title (otherwise we will to straight to the edit form.) if (empty($title) && strpos(get_url_scheme($url), 'http') !== false) { // Short timeout to keep the application responsive - list($headers, $data) = get_http_url($url, 4); - // FIXME: Decode charset according to specified in either 1) HTTP response headers or 2) in html + list($headers, $content) = get_http_response($url, 4); if (strpos($headers[0], '200 OK') !== false) { - // Look for charset in html header. - preg_match('##Usi', $data, $meta); - - // If found, extract encoding. - if (!empty($meta[0])) { - // Get encoding specified in header. - preg_match('#charset="?(.*)"#si', $meta[0], $enc); - // If charset not found, use utf-8. - $html_charset = (!empty($enc[1])) ? strtolower($enc[1]) : 'utf-8'; - } - else { - $html_charset = 'utf-8'; - } - - // Extract title - $title = html_extract_title($data); - if (!empty($title)) { - // Re-encode title in utf-8 if necessary. - $title = ($html_charset == 'iso-8859-1') ? utf8_encode($title) : $title; + // Retrieve charset. + $charset = get_charset($headers, $content); + // Extract title. + $title = html_extract_title($content); + // Re-encode title in utf-8 if necessary. + if (! empty($title) && $charset != 'utf-8') { + $title = mb_convert_encoding($title, $charset, 'utf-8'); } } } + if ($url == '') { $url = '?' . smallHash($linkdate); $title = 'Note: '; } + $link = array( 'linkdate' => $linkdate, 'title' => $title, @@ -2314,11 +2297,11 @@ function genThumbnail() else // This is a flickr page (html) { // Get the flickr html page. - list($headers, $data) = get_http_url($url, 20); + list($headers, $content) = get_http_response($url, 20); if (strpos($headers[0], '200 OK') !== false) { // flickr now nicely provides the URL of the thumbnail in each flickr page. - preg_match('! if ($imageurl=='') { - preg_match('! tag on that page // http://www.ted.com/talks/mikko_hypponen_fighting_viruses_defending_the_net.html // - list($headers, $data) = get_http_url($url, 5); + list($headers, $content) = get_http_response($url, 5); if (strpos($headers[0], '200 OK') !== false) { // Extract the link to the thumbnail - preg_match('!link rel="image_src" href="(http://images.ted.com/images/ted/.+_\d+x\d+\.jpg)"!',$data,$matches); + preg_match('!link rel="image_src" href="(http://images.ted.com/images/ted/.+_\d+x\d+\.jpg)"!', $content, $matches); if (!empty($matches[1])) { // Let's download the image. $imageurl=$matches[1]; // No control on image size, so wait long enough - list($headers, $data) = get_http_url($imageurl, 20); + list($headers, $content) = get_http_response($imageurl, 20); if (strpos($headers[0], '200 OK') !== false) { $filepath=$GLOBALS['config']['CACHEDIR'].'/'.$thumbname; - file_put_contents($filepath,$data); // Save image to cache. + file_put_contents($filepath, $content); // Save image to cache. if (resizeImage($filepath)) { header('Content-Type: image/jpeg'); @@ -2398,18 +2383,19 @@ function genThumbnail() // There is no thumbnail available for xkcd comics, so download the whole image and resize it. // http://xkcd.com/327/ // <BLABLA> - list($headers, $data) = get_http_url($url, 5); + list($headers, $content) = get_http_response($url, 5); if (strpos($headers[0], '200 OK') !== false) { // Extract the link to the thumbnail - preg_match('!'; + $default = 'default'; + $this->assertEquals('headers', get_charset($headers, $html, $default)); + $this->assertEquals('html', get_charset(array(), $html, $default)); + $this->assertEquals($default, get_charset(array(), '', $default)); + $this->assertEquals('utf-8', get_charset(array(), '')); + } + + /** + * Test headers_extract_charset() when the charset is found. + */ + public function testHeadersExtractExistentCharset() + { + $charset = 'x-MacCroatian'; + $headers = array('Content-Type' => 'text/html; charset='. $charset); + $this->assertEquals(strtolower($charset), headers_extract_charset($headers)); + } + + /** + * Test headers_extract_charset() when the charset is not found. + */ + public function testHeadersExtractNonExistentCharset() + { + $headers = array(); + $this->assertFalse(headers_extract_charset($headers)); + + $headers = array('Content-Type' => 'text/html'); + $this->assertFalse(headers_extract_charset($headers)); + } + + /** + * Test html_extract_charset() when the charset is found. + */ + public function testHtmlExtractExistentCharset() + { + $charset = 'x-MacCroatian'; + $html = 'stuff2'; + $this->assertEquals(strtolower($charset), html_extract_charset($html)); + } + + /** + * Test html_extract_charset() when the charset is not found. + */ + public function testHtmlExtractNonExistentCharset() + { + $html = 'stuff'; + $this->assertFalse(html_extract_charset($html)); + $html = 'stuff'; + $this->assertFalse(html_extract_charset($html)); + } +} diff --git a/tests/Url/UrlTest.php b/tests/Url/UrlTest.php index af6daaa4..425327ed 100644 --- a/tests/Url/UrlTest.php +++ b/tests/Url/UrlTest.php @@ -156,4 +156,22 @@ class UrlTest extends PHPUnit_Framework_TestCase $this->assertEquals($strOn, add_trailing_slash($strOn)); $this->assertEquals($strOn, add_trailing_slash($strOff)); } + + /** + * Test valid HTTP url. + */ + function testUrlIsHttp() + { + $url = new Url(self::$baseUrl); + $this->assertTrue($url->isHttp()); + } + + /** + * Test non HTTP url. + */ + function testUrlIsNotHttp() + { + $url = new Url('ftp://save.tld/mysave'); + $this->assertFalse($url->isHttp()); + } } -- cgit v1.2.3