10 files changed, 369 insertions, 40 deletions

diff --git a/application/HttpUtils.php b/application/HttpUtils.php
index af7cb371..2e0792f9 100644
--- a/application/HttpUtils.php
+++ b/application/HttpUtils.php
@@ -27,7 +27,9 @@
 function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
 {
     $urlObj = new Url($url);
-    if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
+    $cleanUrl = $urlObj->idnToAscii();
+
+    if (! filter_var($cleanUrl, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
         return array(array(0 => 'Invalid HTTP Url'), false);
     }
 
@@ -35,22 +37,27 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
         'http' => array(
             'method' => 'GET',
             'timeout' => $timeout,
-            'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)'
-                         .' Gecko/20100101 Firefox/23.0',
-            'request_fulluri' => true,
+            'user_agent' => 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:45.0)'
+                         .' Gecko/20100101 Firefox/45.0',
+            'accept_language' => substr(setlocale(LC_COLLATE, 0), 0, 2) . ',en-US;q=0.7,en;q=0.3',
         )
     );
 
-    $context = stream_context_create($options);
     stream_context_set_default($options);
+    list($headers, $finalUrl) = get_redirected_headers($cleanUrl);
+    if (! $headers || strpos($headers[0], '200 OK') === false) {
+        $options['http']['request_fulluri'] = true;
+        stream_context_set_default($options);
+        list($headers, $finalUrl) = get_redirected_headers($cleanUrl);
+    }
 
-    list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup());
     if (! $headers || strpos($headers[0], '200 OK') === false) {
         return array($headers, false);
     }
 
     try {
         // TODO: catch Exception in calling code (thumbnailer)
+        $context = stream_context_create($options);
         $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
     } catch (Exception $exc) {
         return array(array(0 => 'HTTP Error'), $exc->getMessage());
@@ -60,16 +67,19 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
 }
 
 /**
- * Retrieve HTTP headers, following n redirections (temporary and permanent).
+ * Retrieve HTTP headers, following n redirections (temporary and permanent ones).
  *
- * @param string $url initial URL to reach.
- * @param int $redirectionLimit max redirection follow..
+ * @param string $url              initial URL to reach.
+ * @param int    $redirectionLimit max redirection follow.
  *
- * @return array
+ * @return array HTTP headers, or false if it failed.
  */
 function get_redirected_headers($url, $redirectionLimit = 3)
 {
     $headers = get_headers($url, 1);
+    if (!empty($headers['location']) && empty($headers['Location'])) {
+        $headers['Location'] = $headers['location'];
+    }
 
     // Headers found, redirection found, and limit not reached.
     if ($redirectionLimit-- > 0
@@ -79,6 +89,7 @@ function get_redirected_headers($url, $redirectionLimit = 3)
 
         $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
         if ($redirection != $url) {
+            $redirection = getAbsoluteUrl($url, $redirection);
             return get_redirected_headers($redirection, $redirectionLimit);
         }
     }
@@ -87,6 +98,35 @@ function get_redirected_headers($url, $redirectionLimit = 3)
 }
 
 /**
+ * Get an absolute URL from a complete one, and another absolute/relative URL.
+ *
+ * @param string $originalUrl The original complete URL.
+ * @param string $newUrl      The new one, absolute or relative.
+ *
+ * @return string Final URL:
+ *   - $newUrl if it was already an absolute URL.
+ *   - if it was relative, absolute URL from $originalUrl path.
+ */
+function getAbsoluteUrl($originalUrl, $newUrl)
+{
+    $newScheme = parse_url($newUrl, PHP_URL_SCHEME);
+    // Already an absolute URL.
+    if (!empty($newScheme)) {
+        return $newUrl;
+    }
+
+    $parts = parse_url($originalUrl);
+    $final = $parts['scheme'] .'://'. $parts['host'];
+    $final .= (!empty($parts['port'])) ? $parts['port'] : '';
+    $final .= '/';
+    if ($newUrl[0] != '/') {
+        $final .= substr(ltrim($parts['path'], '/'), 0, strrpos($parts['path'], '/'));
+    }
+    $final .= ltrim($newUrl, '/');
+    return $final;
+}
+
+/**
  * Returns the server's base URL: scheme://domain.tld[:port]
  *
  * @param array $server the $_SERVER array
@@ -153,7 +193,7 @@ function server_url($server)
 function index_url($server)
 {
     $scriptname = $server['SCRIPT_NAME'];
-    if (endswith($scriptname, 'index.php')) {
+    if (endsWith($scriptname, 'index.php')) {
         $scriptname = substr($scriptname, 0, -9);
     }
     return server_url($server) . $scriptname;
diff --git a/application/LinkDB.php b/application/LinkDB.php
index 4c1a45b5..b1072e07 100644
--- a/application/LinkDB.php
+++ b/application/LinkDB.php
@@ -66,21 +66,39 @@ class LinkDB implements Iterator, Countable, ArrayAccess
     private $_redirector;
 
     /**
+     * Set this to `true` to urlencode link behind redirector link, `false` to leave it untouched.
+     *
+     * Example:
+     *   anonym.to needs clean URL while dereferer.org needs urlencoded URL.
+     *
+     * @var boolean $redirectorEncode parameter: true or false
+     */
+    private $redirectorEncode;
+
+    /**
      * Creates a new LinkDB
      *
      * Checks if the datastore exists; else, attempts to create a dummy one.
      *
-     * @param string  $datastore       datastore file path.
-     * @param boolean $isLoggedIn      is the user logged in?
-     * @param boolean $hidePublicLinks if true all links are private.
-     * @param string  $redirector      link redirector set in user settings.
+     * @param string  $datastore        datastore file path.
+     * @param boolean $isLoggedIn       is the user logged in?
+     * @param boolean $hidePublicLinks  if true all links are private.
+     * @param string  $redirector       link redirector set in user settings.
+     * @param boolean $redirectorEncode Enable urlencode on redirected urls (default: true).
      */
-    function __construct($datastore, $isLoggedIn, $hidePublicLinks, $redirector = '')
+    function __construct(
+        $datastore,
+        $isLoggedIn,
+        $hidePublicLinks,
+        $redirector = '',
+        $redirectorEncode = true
+    )
     {
         $this->_datastore = $datastore;
         $this->_loggedIn = $isLoggedIn;
         $this->_hidePublicLinks = $hidePublicLinks;
         $this->_redirector = $redirector;
+        $this->redirectorEncode = $redirectorEncode === true;
         $this->_checkDB();
         $this->_readDB();
     }
@@ -278,7 +296,12 @@ You use the community supported version of the original Shaarli project, by Seba
 
             // Do not use the redirector for internal links (Shaarli note URL starting with a '?').
             if (!empty($this->_redirector) && !startsWith($link['url'], '?')) {
-                $link['real_url'] = $this->_redirector . urlencode($link['url']);
+                $link['real_url'] = $this->_redirector;
+                if ($this->redirectorEncode) {
+                    $link['real_url'] .= urlencode(unescape($link['url']));
+                } else {
+                    $link['real_url'] .= $link['url'];
+                }
             }
             else {
                 $link['real_url'] = $link['url'];
diff --git a/application/LinkFilter.php b/application/LinkFilter.php
index 5e0d8015..e693b284 100644
--- a/application/LinkFilter.php
+++ b/application/LinkFilter.php
@@ -322,7 +322,7 @@ class LinkFilter
         $tagsOut = $casesensitive ? $tags : mb_convert_case($tags, MB_CASE_LOWER, 'UTF-8');
         $tagsOut = str_replace(',', ' ', $tagsOut);
 
-        return array_filter(explode(' ', trim($tagsOut)), 'strlen');
+        return array_values(array_filter(explode(' ', trim($tagsOut)), 'strlen'));
     }
 }
 
diff --git a/application/LinkUtils.php b/application/LinkUtils.php
index d8dc8b5e..da04ca97 100644
--- a/application/LinkUtils.php
+++ b/application/LinkUtils.php
@@ -9,8 +9,8 @@
  */
 function html_extract_title($html)
 {
-    if (preg_match('!<title>(.*?)</title>!is', $html, $matches)) {
-        return trim(str_replace("\n", ' ', $matches[1]));
+    if (preg_match('!<title.*?>(.*?)</title>!is', $html, $matches)) {
+        return trim(str_replace("\n", '', $matches[1]));
     }
     return false;
 }
@@ -70,10 +70,26 @@ function headers_extract_charset($headers)
 function html_extract_charset($html)
 {
     // Get encoding specified in HTML header.
-    preg_match('#<meta .*charset="?([^">/]+)"? */?>#Usi', $html, $enc);
+    preg_match('#<meta .*charset=["\']?([^";\'>/]+)["\']? */?>#Usi', $html, $enc);
     if (!empty($enc[1])) {
         return strtolower($enc[1]);
     }
 
     return false;
 }
+
+/**
+ * Count private links in given linklist.
+ *
+ * @param array $links Linklist.
+ *
+ * @return int Number of private links.
+ */
+function count_private($links)
+{
+    $cpt = 0;
+    foreach ($links as $link) {
+        $cpt = $link['private'] == true ? $cpt + 1 : $cpt;
+    }
+    return $cpt;
+}
diff --git a/application/NetscapeBookmarkUtils.php b/application/NetscapeBookmarkUtils.php
new file mode 100644
index 00000000..fdbb0ad7
--- /dev/null
+++ b/application/NetscapeBookmarkUtils.php
@@ -0,0 +1,54 @@
+<?php
+
+/**
+ * Utilities to import and export bookmarks using the Netscape format
+ */
+class NetscapeBookmarkUtils
+{
+
+    /**
+     * Filters links and adds Netscape-formatted fields
+     *
+     * Added fields:
+     * - timestamp  link addition date, using the Unix epoch format
+     * - taglist    comma-separated tag list
+     *
+     * @param LinkDB $linkDb         Link datastore
+     * @param string $selection      Which links to export: (all|private|public)
+     * @param bool   $prependNoteUrl Prepend note permalinks with the server's URL
+     * @param string $indexUrl       Absolute URL of the Shaarli index page
+     *
+     * @throws Exception Invalid export selection
+     *
+     * @return array The links to be exported, with additional fields
+     */
+    public static function filterAndFormat($linkDb, $selection, $prependNoteUrl, $indexUrl)
+    {
+        // see tpl/export.html for possible values
+        if (! in_array($selection, array('all', 'public', 'private'))) {
+            throw new Exception('Invalid export selection: "'.$selection.'"');
+        }
+
+        $bookmarkLinks = array();
+
+        foreach ($linkDb as $link) {
+            if ($link['private'] != 0 && $selection == 'public') {
+                continue;
+            }
+            if ($link['private'] == 0 && $selection == 'private') {
+                continue;
+            }
+            $date = DateTime::createFromFormat(LinkDB::LINK_DATE_FORMAT, $link['linkdate']);
+            $link['timestamp'] = $date->getTimestamp();
+            $link['taglist'] = str_replace(' ', ',', $link['tags']);
+
+            if (startsWith($link['url'], '?') && $prependNoteUrl) {
+                $link['url'] = $indexUrl . $link['url'];
+            }
+
+            $bookmarkLinks[] = $link;
+        }
+
+        return $bookmarkLinks;
+    }
+}
diff --git a/application/PageBuilder.php b/application/PageBuilder.php
new file mode 100644
index 00000000..82580787
--- /dev/null
+++ b/application/PageBuilder.php
@@ -0,0 +1,145 @@
+<?php
+
+/**
+ * This class is in charge of building the final page.
+ * (This is basically a wrapper around RainTPL which pre-fills some fields.)
+ * $p = new PageBuilder();
+ * $p->assign('myfield','myvalue');
+ * $p->renderPage('mytemplate');
+ */
+class PageBuilder
+{
+    /**
+     * @var RainTPL RainTPL instance.
+     */
+    private $tpl;
+
+    /**
+     * PageBuilder constructor.
+     * $tpl is initialized at false for lazy loading.
+     */
+    function __construct()
+    {
+        $this->tpl = false;
+    }
+
+    /**
+     * Initialize all default tpl tags.
+     */
+    private function initialize()
+    {
+        $this->tpl = new RainTPL();
+
+        try {
+            $version = ApplicationUtils::checkUpdate(
+                shaarli_version,
+                $GLOBALS['config']['UPDATECHECK_FILENAME'],
+                $GLOBALS['config']['UPDATECHECK_INTERVAL'],
+                $GLOBALS['config']['ENABLE_UPDATECHECK'],
+                isLoggedIn(),
+                $GLOBALS['config']['UPDATECHECK_BRANCH']
+            );
+            $this->tpl->assign('newVersion', escape($version));
+            $this->tpl->assign('versionError', '');
+
+        } catch (Exception $exc) {
+            logm($GLOBALS['config']['LOG_FILE'], $_SERVER['REMOTE_ADDR'], $exc->getMessage());
+            $this->tpl->assign('newVersion', '');
+            $this->tpl->assign('versionError', escape($exc->getMessage()));
+        }
+
+        $this->tpl->assign('feedurl', escape(index_url($_SERVER)));
+        $searchcrits = ''; // Search criteria
+        if (!empty($_GET['searchtags'])) {
+            $searchcrits .= '&searchtags=' . urlencode($_GET['searchtags']);
+        }
+        if (!empty($_GET['searchterm'])) {
+            $searchcrits .= '&searchterm=' . urlencode($_GET['searchterm']);
+        }
+        $this->tpl->assign('searchcrits', $searchcrits);
+        $this->tpl->assign('source', index_url($_SERVER));
+        $this->tpl->assign('version', shaarli_version);
+        $this->tpl->assign('scripturl', index_url($_SERVER));
+        $this->tpl->assign('pagetitle', 'Shaarli');
+        $this->tpl->assign('privateonly', !empty($_SESSION['privateonly'])); // Show only private links?
+        if (!empty($GLOBALS['title'])) {
+            $this->tpl->assign('pagetitle', $GLOBALS['title']);
+        }
+        if (!empty($GLOBALS['titleLink'])) {
+            $this->tpl->assign('titleLink', $GLOBALS['titleLink']);
+        }
+        if (!empty($GLOBALS['pagetitle'])) {
+            $this->tpl->assign('pagetitle', $GLOBALS['pagetitle']);
+        }
+        $this->tpl->assign('shaarlititle', empty($GLOBALS['title']) ? 'Shaarli': $GLOBALS['title']);
+        if (!empty($GLOBALS['plugin_errors'])) {
+            $this->tpl->assign('plugin_errors', $GLOBALS['plugin_errors']);
+        }
+    }
+
+    /**
+     * The following assign() method is basically the same as RainTPL (except lazy loading)
+     *
+     * @param string $placeholder Template placeholder.
+     * @param mixed  $value       Value to assign.
+     */
+    public function assign($placeholder, $value)
+    {
+        // Lazy initialization
+        if ($this->tpl === false) {
+            $this->initialize();
+        }
+        $this->tpl->assign($placeholder, $value);
+    }
+
+    /**
+     * Assign an array of data to the template builder.
+     *
+     * @param array $data Data to assign.
+     *
+     * @return false if invalid data.
+     */
+    public function assignAll($data)
+    {
+        // Lazy initialization
+        if ($this->tpl === false) {
+            $this->initialize();
+        }
+
+        if (empty($data) || !is_array($data)){
+            return false;
+        }
+
+        foreach ($data as $key => $value) {
+            $this->assign($key, $value);
+        }
+    }
+
+    /**
+     * Render a specific page (using a template file).
+     * e.g. $pb->renderPage('picwall');
+     *
+     * @param string $page Template filename (without extension).
+     */
+    public function renderPage($page)
+    {
+        // Lazy initialization
+        if ($this->tpl===false) {
+            $this->initialize();
+        }
+        $this->tpl->draw($page);
+    }
+
+    /**
+     * Render a 404 page (uses the template : tpl/404.tpl)
+     * usage : $PAGE->render404('The link was deleted')
+     *
+     * @param string $message A messate to display what is not found
+     */
+    public function render404($message = 'The page you are trying to reach does not exist or has been deleted.')
+    {
+        header($_SERVER['SERVER_PROTOCOL'] . ' 404 Not Found');
+        $this->tpl->assign('error_message', $message);
+        $this->renderPage('404');
+    }
+}
diff --git a/application/Router.php b/application/Router.php
index a1e594a0..2c3934b0 100644
--- a/application/Router.php
+++ b/application/Router.php
@@ -63,19 +63,19 @@ class Router
             return self::$PAGE_LINKLIST;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_LOGIN) && $loggedIn === false) {
+        if (startsWith($query, 'do='. self::$PAGE_LOGIN) && $loggedIn === false) {
             return self::$PAGE_LOGIN;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_PICWALL)) {
+        if (startsWith($query, 'do='. self::$PAGE_PICWALL)) {
             return self::$PAGE_PICWALL;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_TAGCLOUD)) {
+        if (startsWith($query, 'do='. self::$PAGE_TAGCLOUD)) {
             return self::$PAGE_TAGCLOUD;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_OPENSEARCH)) {
+        if (startsWith($query, 'do='. self::$PAGE_OPENSEARCH)) {
             return self::$PAGE_OPENSEARCH;
         }
 
@@ -96,23 +96,23 @@ class Router
             return self::$PAGE_LINKLIST;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_TOOLS)) {
+        if (startsWith($query, 'do='. self::$PAGE_TOOLS)) {
             return self::$PAGE_TOOLS;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_CHANGEPASSWORD)) {
+        if (startsWith($query, 'do='. self::$PAGE_CHANGEPASSWORD)) {
             return self::$PAGE_CHANGEPASSWORD;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_CONFIGURE)) {
+        if (startsWith($query, 'do='. self::$PAGE_CONFIGURE)) {
             return self::$PAGE_CONFIGURE;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_CHANGETAG)) {
+        if (startsWith($query, 'do='. self::$PAGE_CHANGETAG)) {
             return self::$PAGE_CHANGETAG;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_ADDLINK)) {
+        if (startsWith($query, 'do='. self::$PAGE_ADDLINK)) {
             return self::$PAGE_ADDLINK;
         }
 
@@ -120,19 +120,19 @@ class Router
             return self::$PAGE_EDITLINK;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_EXPORT)) {
+        if (startsWith($query, 'do='. self::$PAGE_EXPORT)) {
             return self::$PAGE_EXPORT;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_IMPORT)) {
+        if (startsWith($query, 'do='. self::$PAGE_IMPORT)) {
             return self::$PAGE_IMPORT;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_PLUGINSADMIN)) {
+        if (startsWith($query, 'do='. self::$PAGE_PLUGINSADMIN)) {
             return self::$PAGE_PLUGINSADMIN;
         }
 
-        if (startswith($query, 'do='. self::$PAGE_SAVE_PLUGINSADMIN)) {
+        if (startsWith($query, 'do='. self::$PAGE_SAVE_PLUGINSADMIN)) {
             return self::$PAGE_SAVE_PLUGINSADMIN;
         }
 
diff --git a/application/TimeZone.php b/application/TimeZone.php
index e363d90a..26f2232d 100644
--- a/application/TimeZone.php
+++ b/application/TimeZone.php
@@ -101,10 +101,6 @@ function generateTimeZoneForm($preselectedTimezone='')
  */
 function isTimeZoneValid($continent, $city)
 {
-    if ($continent == 'UTC' && $city == 'UTC') {
-        return true;
-    }
-
     return in_array(
         $continent.'/'.$city,
         timezone_identifiers_list()
diff --git a/application/Url.php b/application/Url.php
index af38c4d9..c166ff6e 100644
--- a/application/Url.php
+++ b/application/Url.php
@@ -62,7 +62,21 @@ function add_trailing_slash($url)
 {
     return $url . (!endsWith($url, '/') ? '/' : '');
 }
+/**
+ * Converts an URL with an IDN host to a ASCII one.
+ *
+ * @param string $url Input URL.
+ *
+ * @return string converted URL.
+ */
+function url_with_idn_to_ascii($url)
+{
+    $parts = parse_url($url);
+    $parts['host'] = idn_to_ascii($parts['host']);
 
+    $httpUrl = new \http\Url($parts);
+    return $httpUrl->toString();
+}
 /**
  * URL representation and cleanup utilities
  *
@@ -85,6 +99,7 @@ class Url
         'action_type_map=',
         'fb_',
         'fb=',
+        'PHPSESSID=',
 
         // Scoop.it
         '__scoop',
@@ -221,6 +236,22 @@ class Url
     }
 
     /**
+     * Converts an URL with an International Domain Name host to a ASCII one.
+     * This requires PHP-intl. If it's not available, just returns this->cleanup().
+     *
+     * @return string converted cleaned up URL.
+     */
+    public function idnToAscii()
+    {
+        $out = $this->cleanup();
+        if (! function_exists('idn_to_ascii') || ! isset($this->parts['host'])) {
+            return $out;
+        }
+        $asciiHost = idn_to_ascii($this->parts['host']);
+        return str_replace($this->parts['host'], $asciiHost, $out);
+    }
+
+    /**
      * Get URL scheme.
      *
      * @return string the URL scheme or false if none is provided.
@@ -233,6 +264,18 @@ class Url
     }
 
     /**
+     * Get URL host.
+     *
+     * @return string the URL host or false if none is provided.
+     */
+    public function getHost() {
+        if (empty($this->parts['host'])) {
+            return false;
+        }
+        return $this->parts['host'];
+    }
+
+    /**
      * Test if the Url is an HTTP one.
      *
      * @return true is HTTP, false otherwise.
diff --git a/application/Utils.php b/application/Utils.php
index 5b8ca508..da521cce 100644
--- a/application/Utils.php
+++ b/application/Utils.php
@@ -41,8 +41,14 @@ function smallHash($text)
 
 /**
  * Tells if a string start with a substring
+ *
+ * @param string $haystack Given string.
+ * @param string $needle   String to search at the beginning of $haystack.
+ * @param bool   $case     Case sensitive.
+ *
+ * @return bool True if $haystack starts with $needle.
  */
-function startsWith($haystack, $needle, $case=true)
+function startsWith($haystack, $needle, $case = true)
 {
     if ($case) {
         return (strcmp(substr($haystack, 0, strlen($needle)), $needle) === 0);
@@ -52,8 +58,14 @@ function startsWith($haystack, $needle, $case=true)
 
 /**
  * Tells if a string ends with a substring
+ *
+ * @param string $haystack Given string.
+ * @param string $needle   String to search at the end of $haystack.
+ * @param bool   $case     Case sensitive.
+ *
+ * @return bool True if $haystack ends with $needle.
  */
-function endsWith($haystack, $needle, $case=true)
+function endsWith($haystack, $needle, $case = true)
 {
     if ($case) {
         return (strcmp(substr($haystack, strlen($haystack) - strlen($needle)), $needle) === 0);