Fixes #410 - Retrieve title fails in multiple cases

  * `get_http_url()` renamed to `get_http_response()`.
  * Use the same HTTP context to retrieve response headers and content.
  * Follow HTTP 301 and 302 redirections to retrieve the title (default max 3 redirections).
  * Add `LinkUtils` to extract titles and charset.
  * Try to retrieve charset from HTTP headers first (new), then HTML content.
  * Use mb_string to re-encode title if necessary.

4 files changed, 132 insertions, 9 deletions

diff --git a/application/ApplicationUtils.php b/application/ApplicationUtils.php
index 274331e1..978fc9da 100644
--- a/application/ApplicationUtils.php
+++ b/application/ApplicationUtils.php
@@ -19,7 +19,7 @@ class ApplicationUtils
      */
     public static function getLatestGitVersionCode($url, $timeout=2)
     {
-        list($headers, $data) = get_http_url($url, $timeout);
+        list($headers, $data) = get_http_response($url, $timeout);
 
         if (strpos($headers[0], '200 OK') === false) {
             error_log('Failed to retrieve ' . $url);
diff --git a/application/HttpUtils.php b/application/HttpUtils.php
index 499220c5..e2c1cb47 100644..100755
--- a/application/HttpUtils.php
+++ b/application/HttpUtils.php
@@ -13,7 +13,7 @@
  *  [1] = URL content (downloaded data)
  *
  * Example:
- *  list($headers, $data) = get_http_url('http://sebauvage.net/');
+ *  list($headers, $data) = get_http_response('http://sebauvage.net/');
  *  if (strpos($headers[0], '200 OK') !== false) {
  *      echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
  *  } else {
@@ -24,31 +24,66 @@
  * @see http://php.net/manual/en/function.stream-context-create.php
  * @see http://php.net/manual/en/function.get-headers.php
  */
-function get_http_url($url, $timeout = 30, $maxBytes = 4194304)
+function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
 {
+    $urlObj = new Url($url);
+    if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
+        return array(array(0 => 'Invalid HTTP Url'), false);
+    }
+
     $options = array(
         'http' => array(
             'method' => 'GET',
             'timeout' => $timeout,
             'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)'
-                         .' Gecko/20100101 Firefox/23.0'
+                         .' Gecko/20100101 Firefox/23.0',
+            'request_fulluri' => true,
         )
     );
 
     $context = stream_context_create($options);
+    stream_context_set_default($options);
+
+    list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup());
+    if (! $headers || strpos($headers[0], '200 OK') === false) {
+        return array($headers, false);
+    }
 
     try {
         // TODO: catch Exception in calling code (thumbnailer)
-        $content = file_get_contents($url, false, $context, -1, $maxBytes);
+        $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
     } catch (Exception $exc) {
         return array(array(0 => 'HTTP Error'), $exc->getMessage());
     }
 
-    if (!$content) {
-        return array(array(0 => 'HTTP Error'), '');
+    return array($headers, $content);
+}
+
+/**
+ * Retrieve HTTP headers, following n redirections (temporary and permanent).
+ *
+ * @param string $url initial URL to reach.
+ * @param int $redirectionLimit max redirection follow..
+ *
+ * @return array
+ */
+function get_redirected_headers($url, $redirectionLimit = 3)
+{
+    $headers = get_headers($url, 1);
+
+    // Headers found, redirection found, and limit not reached.
+    if ($redirectionLimit-- > 0
+        && !empty($headers)
+        && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
+        && !empty($headers['Location'])) {
+
+        $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
+        if ($redirection != $url) {
+            return get_redirected_headers($redirection, $redirectionLimit);
+        }
     }
 
-    return array(get_headers($url, 1), $content);
+    return array($headers, $url);
 }
 
 /**
diff --git a/application/LinkUtils.php b/application/LinkUtils.php
new file mode 100755
index 00000000..26dd6b67
--- /dev/null
+++ b/application/LinkUtils.php
@@ -0,0 +1,79 @@
+<?php
+
+/**
+ * Extract title from an HTML document.
+ *
+ * @param string $html HTML content where to look for a title.
+ *
+ * @return bool|string Extracted title if found, false otherwise.
+ */
+function html_extract_title($html)
+{
+    if (preg_match('!<title>(.*)</title>!is', $html, $matches)) {
+        return trim(str_replace("\n", ' ', $matches[1]));
+    }
+    return false;
+}
+
+/**
+ * Determine charset from downloaded page.
+ * Priority:
+ *   1. HTTP headers (Content type).
+ *   2. HTML content page (tag <meta charset>).
+ *   3. Use a default charset (default: UTF-8).
+ *
+ * @param array  $headers           HTTP headers array.
+ * @param string $htmlContent       HTML content where to look for charset.
+ * @param string $defaultCharset    Default charset to apply if other methods failed.
+ *
+ * @return string Determined charset.
+ */
+function get_charset($headers, $htmlContent, $defaultCharset = 'utf-8')
+{
+    if ($charset = headers_extract_charset($headers)) {
+        return $charset;
+    }
+
+    if ($charset = html_extract_charset($htmlContent)) {
+        return $charset;
+    }
+
+    return $defaultCharset;
+}
+
+/**
+ * Extract charset from HTTP headers if it's defined.
+ *
+ * @param array $headers HTTP headers array.
+ *
+ * @return bool|string Charset string if found (lowercase), false otherwise.
+ */
+function headers_extract_charset($headers)
+{
+    if (! empty($headers['Content-Type']) && strpos($headers['Content-Type'], 'charset=') !== false) {
+        preg_match('/charset="?([^; ]+)/i', $headers['Content-Type'], $match);
+        if (! empty($match[1])) {
+            return strtolower(trim($match[1]));
+        }
+    }
+
+    return false;
+}
+
+/**
+ * Extract charset HTML content (tag <meta charset>).
+ *
+ * @param string $html HTML content where to look for charset.
+ *
+ * @return bool|string Charset string if found, false otherwise.
+ */
+function html_extract_charset($html)
+{
+    // Get encoding specified in HTML header.
+    preg_match('#<meta .*charset="?([^">/]+)"? */?>#Usi', $html, $enc);
+    if (!empty($enc[1])) {
+        return strtolower($enc[1]);
+    }
+
+    return false;
+}
diff --git a/application/Url.php b/application/Url.php
index d80c9c58..a4ac2e73 100644..100755
--- a/application/Url.php
+++ b/application/Url.php
@@ -118,7 +118,7 @@ class Url
      */
     public function __construct($url)
     {
-        $this->parts = parse_url($url);
+        $this->parts = parse_url(trim($url));
 
         if (!empty($url) && empty($this->parts['scheme'])) {
             $this->parts['scheme'] = 'http';
@@ -201,4 +201,13 @@ class Url
         }
         return $this->parts['scheme'];
     }
+
+    /**
+     * Test if the Url is an HTTP one.
+     *
+     * @return true is HTTP, false otherwise.
+     */
+    public function isHttp() {
+        return strpos(strtolower($this->parts['scheme']), 'http') !== false;
+    }
 }