5 files changed, 78 insertions, 42 deletions

diff --git a/application/HttpUtils.php b/application/HttpUtils.php
index ec54dcd4..83a4c5e2 100644
--- a/application/HttpUtils.php
+++ b/application/HttpUtils.php
@@ -3,9 +3,11 @@
  * GET an HTTP URL to retrieve its content
  * Uses the cURL library or a fallback method 
  *
- * @param string $url      URL to get (http://...)
- * @param int    $timeout  network timeout (in seconds)
- * @param int    $maxBytes maximum downloaded bytes (default: 4 MiB)
+ * @param string          $url               URL to get (http://...)
+ * @param int             $timeout           network timeout (in seconds)
+ * @param int             $maxBytes          maximum downloaded bytes (default: 4 MiB)
+ * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
+ *                                           Can be used to add download conditions on the headers (response code, content type, etc.).
  *
  * @return array HTTP response headers, downloaded content
  *
@@ -29,7 +31,7 @@
  * @see http://stackoverflow.com/q/9183178
  * @see http://stackoverflow.com/q/1462720
  */
-function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
+function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null)
 {
     $urlObj = new Url($url);
     $cleanUrl = $urlObj->idnToAscii();
@@ -75,6 +77,10 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
     curl_setopt($ch, CURLOPT_TIMEOUT,           $timeout);
     curl_setopt($ch, CURLOPT_USERAGENT,         $userAgent);
 
+    if (is_callable($curlWriteFunction)) {
+        curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
+    }
+
     // Max download size management
     curl_setopt($ch, CURLOPT_BUFFERSIZE,        1024*16);
     curl_setopt($ch, CURLOPT_NOPROGRESS,        false);
@@ -302,6 +308,13 @@ function server_url($server)
                 $port = $server['HTTP_X_FORWARDED_PORT'];
             }
 
+            // This is a workaround for proxies that don't forward the scheme properly.
+            // Connecting over port 443 has to be in HTTPS.
+            // See https://github.com/shaarli/Shaarli/issues/1022
+            if ($port == '443') {
+                $scheme = 'https';
+            }
+
             if (($scheme == 'http' && $port != '80')
                 || ($scheme == 'https' && $port != '443')
             ) {
diff --git a/application/Languages.php b/application/Languages.php
index 357c7524..3eb3388f 100644
--- a/application/Languages.php
+++ b/application/Languages.php
@@ -69,6 +69,8 @@ class Languages
     {
         $this->conf = $conf;
         $confLanguage = $this->conf->get('translation.language', 'auto');
+        // Auto mode or invalid parameter, use the detected language.
+        // If the detected language is invalid, it doesn't matter, it will use English.
         if ($confLanguage === 'auto' || ! $this->isValidLanguage($confLanguage)) {
             $this->language = substr($language, 0, 5);
         } else {
diff --git a/application/LinkUtils.php b/application/LinkUtils.php
index e3d95d08..3705f7e9 100644
--- a/application/LinkUtils.php
+++ b/application/LinkUtils.php
@@ -1,60 +1,81 @@
 <?php
 
 /**
- * Extract title from an HTML document.
+ * Get cURL callback function for CURLOPT_WRITEFUNCTION
  *
- * @param string $html HTML content where to look for a title.
+ * @param string $charset     to extract from the downloaded page (reference)
+ * @param string $title       to extract from the downloaded page (reference)
+ * @param string $curlGetInfo Optionnaly overrides curl_getinfo function
  *
- * @return bool|string Extracted title if found, false otherwise.
+ * @return Closure
  */
-function html_extract_title($html)
+function get_curl_download_callback(&$charset, &$title, $curlGetInfo = 'curl_getinfo')
 {
-    if (preg_match('!<title.*?>(.*?)</title>!is', $html, $matches)) {
-        return trim(str_replace("\n", '', $matches[1]));
-    }
-    return false;
+    /**
+     * cURL callback function for CURLOPT_WRITEFUNCTION (called during the download).
+     *
+     * While downloading the remote page, we check that the HTTP code is 200 and content type is 'html/text'
+     * Then we extract the title and the charset and stop the download when it's done.
+     *
+     * @param resource $ch   cURL resource
+     * @param string   $data chunk of data being downloaded
+     *
+     * @return int|bool length of $data or false if we need to stop the download
+     */
+    return function(&$ch, $data) use ($curlGetInfo, &$charset, &$title) {
+        $responseCode = $curlGetInfo($ch, CURLINFO_RESPONSE_CODE);
+        if (!empty($responseCode) && $responseCode != 200) {
+            return false;
+        }
+        $contentType = $curlGetInfo($ch, CURLINFO_CONTENT_TYPE);
+        if (!empty($contentType) && strpos($contentType, 'text/html') === false) {
+            return false;
+        }
+        if (empty($charset)) {
+            $charset = header_extract_charset($contentType);
+        }
+        if (empty($charset)) {
+            $charset = html_extract_charset($data);
+        }
+        if (empty($title)) {
+            $title = html_extract_title($data);
+        }
+        // We got everything we want, stop the download.
+        if (!empty($responseCode) && !empty($contentType) && !empty($charset) && !empty($title)) {
+            return false;
+        }
+
+        return strlen($data);
+    };
 }
 
 /**
- * Determine charset from downloaded page.
- * Priority:
- *   1. HTTP headers (Content type).
- *   2. HTML content page (tag <meta charset>).
- *   3. Use a default charset (default: UTF-8).
+ * Extract title from an HTML document.
  *
- * @param array  $headers           HTTP headers array.
- * @param string $htmlContent       HTML content where to look for charset.
- * @param string $defaultCharset    Default charset to apply if other methods failed.
+ * @param string $html HTML content where to look for a title.
  *
- * @return string Determined charset.
+ * @return bool|string Extracted title if found, false otherwise.
  */
-function get_charset($headers, $htmlContent, $defaultCharset = 'utf-8')
+function html_extract_title($html)
 {
-    if ($charset = headers_extract_charset($headers)) {
-        return $charset;
-    }
-
-    if ($charset = html_extract_charset($htmlContent)) {
-        return $charset;
+    if (preg_match('!<title.*?>(.*?)</title>!is', $html, $matches)) {
+        return trim(str_replace("\n", '', $matches[1]));
     }
-
-    return $defaultCharset;
+    return false;
 }
 
 /**
- * Extract charset from HTTP headers if it's defined.
+ * Extract charset from HTTP header if it's defined.
  *
- * @param array $headers HTTP headers array.
+ * @param string $header HTTP header Content-Type line.
  *
  * @return bool|string Charset string if found (lowercase), false otherwise.
  */
-function headers_extract_charset($headers)
+function header_extract_charset($header)
 {
-    if (! empty($headers['Content-Type']) && strpos($headers['Content-Type'], 'charset=') !== false) {
-        preg_match('/charset="?([^; ]+)/i', $headers['Content-Type'], $match);
-        if (! empty($match[1])) {
-            return strtolower(trim($match[1]));
-        }
+    preg_match('/charset="?([^; ]+)/i', $header, $match);
+    if (! empty($match[1])) {
+        return strtolower(trim($match[1]));
     }
 
     return false;
diff --git a/application/Url.php b/application/Url.php
index b3759377..21c17ecc 100644
--- a/application/Url.php
+++ b/application/Url.php
@@ -260,7 +260,7 @@ class Url
         if (! function_exists('idn_to_ascii') || ! isset($this->parts['host'])) {
             return $out;
         }
-        $asciiHost = idn_to_ascii($this->parts['host']);
+        $asciiHost = idn_to_ascii($this->parts['host'], 0, INTL_IDNA_VARIANT_UTS46);
         return str_replace($this->parts['host'], $asciiHost, $out);
     }
 
diff --git a/application/config/ConfigPhp.php b/application/config/ConfigPhp.php
index 2f66e8e0..8add8bcd 100644
--- a/application/config/ConfigPhp.php
+++ b/application/config/ConfigPhp.php
@@ -83,10 +83,10 @@ class ConfigPhp implements ConfigIO
 
         $out = array();
         foreach (self::$ROOT_KEYS as $key) {
-            $out[$key] = $GLOBALS[$key];
+            $out[$key] = isset($GLOBALS[$key]) ? $GLOBALS[$key] : '';
         }
-        $out['config'] = $GLOBALS['config'];
-        $out['plugins'] = !empty($GLOBALS['plugins']) ? $GLOBALS['plugins'] : array();
+        $out['config'] = isset($GLOBALS['config']) ? $GLOBALS['config'] : [];
+        $out['plugins'] = isset($GLOBALS['plugins']) ? $GLOBALS['plugins'] : [];
         return $out;
     }