7 files changed, 441 insertions, 119 deletions

diff --git a/application/ApplicationUtils.php b/application/ApplicationUtils.php
index 274331e1..978fc9da 100644
--- a/application/ApplicationUtils.php
+++ b/application/ApplicationUtils.php
@@ -19,7 +19,7 @@ class ApplicationUtils
      */
     public static function getLatestGitVersionCode($url, $timeout=2)
     {
-        list($headers, $data) = get_http_url($url, $timeout);
+        list($headers, $data) = get_http_response($url, $timeout);
 
         if (strpos($headers[0], '200 OK') === false) {
             error_log('Failed to retrieve ' . $url);
diff --git a/application/HttpUtils.php b/application/HttpUtils.php
index 499220c5..e2c1cb47 100644
--- a/application/HttpUtils.php
+++ b/application/HttpUtils.php
@@ -13,7 +13,7 @@
  *  [1] = URL content (downloaded data)
  *
  * Example:
- *  list($headers, $data) = get_http_url('http://sebauvage.net/');
+ *  list($headers, $data) = get_http_response('http://sebauvage.net/');
  *  if (strpos($headers[0], '200 OK') !== false) {
  *      echo 'Data type: '.htmlspecialchars($headers['Content-Type']);
  *  } else {
@@ -24,31 +24,66 @@
  * @see http://php.net/manual/en/function.stream-context-create.php
  * @see http://php.net/manual/en/function.get-headers.php
  */
-function get_http_url($url, $timeout = 30, $maxBytes = 4194304)
+function get_http_response($url, $timeout = 30, $maxBytes = 4194304)
 {
+    $urlObj = new Url($url);
+    if (! filter_var($url, FILTER_VALIDATE_URL) || ! $urlObj->isHttp()) {
+        return array(array(0 => 'Invalid HTTP Url'), false);
+    }
+
     $options = array(
         'http' => array(
             'method' => 'GET',
             'timeout' => $timeout,
             'user_agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:23.0)'
-                         .' Gecko/20100101 Firefox/23.0'
+                         .' Gecko/20100101 Firefox/23.0',
+            'request_fulluri' => true,
         )
     );
 
     $context = stream_context_create($options);
+    stream_context_set_default($options);
+
+    list($headers, $finalUrl) = get_redirected_headers($urlObj->cleanup());
+    if (! $headers || strpos($headers[0], '200 OK') === false) {
+        return array($headers, false);
+    }
 
     try {
         // TODO: catch Exception in calling code (thumbnailer)
-        $content = file_get_contents($url, false, $context, -1, $maxBytes);
+        $content = file_get_contents($finalUrl, false, $context, -1, $maxBytes);
     } catch (Exception $exc) {
         return array(array(0 => 'HTTP Error'), $exc->getMessage());
     }
 
-    if (!$content) {
-        return array(array(0 => 'HTTP Error'), '');
+    return array($headers, $content);
+}
+
+/**
+ * Retrieve HTTP headers, following n redirections (temporary and permanent).
+ *
+ * @param string $url initial URL to reach.
+ * @param int $redirectionLimit max redirection follow..
+ *
+ * @return array
+ */
+function get_redirected_headers($url, $redirectionLimit = 3)
+{
+    $headers = get_headers($url, 1);
+
+    // Headers found, redirection found, and limit not reached.
+    if ($redirectionLimit-- > 0
+        && !empty($headers)
+        && (strpos($headers[0], '301') !== false || strpos($headers[0], '302') !== false)
+        && !empty($headers['Location'])) {
+
+        $redirection = is_array($headers['Location']) ? end($headers['Location']) : $headers['Location'];
+        if ($redirection != $url) {
+            return get_redirected_headers($redirection, $redirectionLimit);
+        }
     }
 
-    return array(get_headers($url, 1), $content);
+    return array($headers, $url);
 }
 
 /**
diff --git a/application/LinkDB.php b/application/LinkDB.php
index f771ac8b..19ca6435 100644
--- a/application/LinkDB.php
+++ b/application/LinkDB.php
@@ -17,8 +17,10 @@
  *  - private:  Is this link private? 0=no, other value=yes
  *  - tags:     tags attached to this entry (separated by spaces)
  *  - title     Title of the link
- *  - url       URL of the link. Can be absolute or relative.
+ *  - url       URL of the link. Used for displayable links (no redirector, relative, etc.).
+ *              Can be absolute or relative.
  *              Relative URLs are permalinks (e.g.'?m-ukcw')
+ *  - real_url  Absolute processed URL.
  *
  * Implements 3 interfaces:
  *  - ArrayAccess: behaves like an associative array;
@@ -332,114 +334,20 @@ You use the community supported version of the original Shaarli project, by Seba
     }
 
     /**
-     * Returns the list of links corresponding to a full-text search
+     * Filter links.
      *
-     * Searches:
-     *  - in the URLs, title and description;
-     *  - are case-insensitive.
+     * @param string $type          Type of filter.
+     * @param mixed  $request       Search request, string or array.
+     * @param bool   $casesensitive Optional: Perform case sensitive filter
+     * @param bool   $privateonly   Optional: Returns private links only if true.
      *
-     * Example:
-     *    print_r($mydb->filterFulltext('hollandais'));
-     *
-     * mb_convert_case($val, MB_CASE_LOWER, 'UTF-8')
-     *  - allows to perform searches on Unicode text
-     *  - see https://github.com/shaarli/Shaarli/issues/75 for examples
-     */
-    public function filterFulltext($searchterms)
-    {
-        // FIXME: explode(' ',$searchterms) and perform a AND search.
-        // FIXME: accept double-quotes to search for a string "as is"?
-        $filtered = array();
-        $search = mb_convert_case($searchterms, MB_CASE_LOWER, 'UTF-8');
-        $keys = array('title', 'description', 'url', 'tags');
-
-        foreach ($this->_links as $link) {
-            $found = false;
-
-            foreach ($keys as $key) {
-                if (strpos(mb_convert_case($link[$key], MB_CASE_LOWER, 'UTF-8'),
-                           $search) !== false) {
-                    $found = true;
-                }
-            }
-
-            if ($found) {
-                $filtered[$link['linkdate']] = $link;
-            }
-        }
-        krsort($filtered);
-        return $filtered;
-    }
-
-    /**
-     * Returns the list of links associated with a given list of tags
-     *
-     * You can specify one or more tags, separated by space or a comma, e.g.
-     *  print_r($mydb->filterTags('linux programming'));
+     * @return array filtered links
      */
-    public function filterTags($tags, $casesensitive=false)
+    public function filter($type, $request, $casesensitive = false, $privateonly = false)
     {
-        // Same as above, we use UTF-8 conversion to handle various graphemes (i.e. cyrillic, or greek)
-        // FIXME: is $casesensitive ever true?
-        $t = str_replace(
-            ',', ' ',
-            ($casesensitive ? $tags : mb_convert_case($tags, MB_CASE_LOWER, 'UTF-8'))
-        );
-
-        $searchtags = explode(' ', $t);
-        $filtered = array();
-
-        foreach ($this->_links as $l) {
-            $linktags = explode(
-                ' ',
-                ($casesensitive ? $l['tags']:mb_convert_case($l['tags'], MB_CASE_LOWER, 'UTF-8'))
-            );
-
-            if (count(array_intersect($linktags, $searchtags)) == count($searchtags)) {
-                $filtered[$l['linkdate']] = $l;
-            }
-        }
-        krsort($filtered);
-        return $filtered;
-    }
-
-
-    /**
-     * Returns the list of articles for a given day, chronologically sorted
-     *
-     * Day must be in the form 'YYYYMMDD' (e.g. '20120125'), e.g.
-     *  print_r($mydb->filterDay('20120125'));
-     */
-    public function filterDay($day)
-    {
-        if (! checkDateFormat('Ymd', $day)) {
-            throw new Exception('Invalid date format');
-        }
-
-        $filtered = array();
-        foreach ($this->_links as $l) {
-            if (startsWith($l['linkdate'], $day)) {
-                $filtered[$l['linkdate']] = $l;
-            }
-        }
-        ksort($filtered);
-        return $filtered;
-    }
-
-    /**
-     * Returns the article corresponding to a smallHash
-     */
-    public function filterSmallHash($smallHash)
-    {
-        $filtered = array();
-        foreach ($this->_links as $l) {
-            if ($smallHash == smallHash($l['linkdate'])) {
-                // Yes, this is ugly and slow
-                $filtered[$l['linkdate']] = $l;
-                return $filtered;
-            }
-        }
-        return $filtered;
+        $linkFilter = new LinkFilter($this->_links);
+        $requestFilter = is_array($request) ? implode(' ', $request) : $request;
+        return $linkFilter->filter($type, trim($requestFilter), $casesensitive, $privateonly);
     }
 
     /**
diff --git a/application/LinkFilter.php b/application/LinkFilter.php
new file mode 100644
index 00000000..cf647371
--- /dev/null
+++ b/application/LinkFilter.php
@@ -0,0 +1,259 @@
+<?php
+
+/**
+ * Class LinkFilter.
+ *
+ * Perform search and filter operation on link data list.
+ */
+class LinkFilter
+{
+    /**
+     * @var string permalinks.
+     */
+    public static $FILTER_HASH   = 'permalink';
+
+    /**
+     * @var string text search.
+     */
+    public static $FILTER_TEXT   = 'fulltext';
+
+    /**
+     * @var string tag filter.
+     */
+    public static $FILTER_TAG    = 'tags';
+
+    /**
+     * @var string filter by day.
+     */
+    public static $FILTER_DAY    = 'FILTER_DAY';
+
+    /**
+     * @var array all available links.
+     */
+    private $links;
+
+    /**
+     * @param array $links initialization.
+     */
+    public function __construct($links)
+    {
+        $this->links = $links;
+    }
+
+    /**
+     * Filter links according to parameters.
+     *
+     * @param string $type          Type of filter (eg. tags, permalink, etc.).
+     * @param string $request       Filter content.
+     * @param bool   $casesensitive Optional: Perform case sensitive filter if true.
+     * @param bool   $privateonly   Optional: Only returns private links if true.
+     *
+     * @return array filtered link list.
+     */
+    public function filter($type, $request, $casesensitive = false, $privateonly = false)
+    {
+        switch($type) {
+            case self::$FILTER_HASH:
+                return $this->filterSmallHash($request);
+                break;
+            case self::$FILTER_TEXT:
+                return $this->filterFulltext($request, $privateonly);
+                break;
+            case self::$FILTER_TAG:
+                return $this->filterTags($request, $casesensitive, $privateonly);
+                break;
+            case self::$FILTER_DAY:
+                return $this->filterDay($request);
+                break;
+            default:
+                return $this->noFilter($privateonly);
+        }
+    }
+
+    /**
+     * Unknown filter, but handle private only.
+     *
+     * @param bool $privateonly returns private link only if true.
+     *
+     * @return array filtered links.
+     */
+    private function noFilter($privateonly = false)
+    {
+        if (! $privateonly) {
+            krsort($this->links);
+            return $this->links;
+        }
+
+        $out = array();
+        foreach ($this->links as $value) {
+            if ($value['private']) {
+                $out[$value['linkdate']] = $value;
+            }
+        }
+
+        krsort($out);
+        return $out;
+    }
+
+    /**
+     * Returns the shaare corresponding to a smallHash.
+     *
+     * @param string $smallHash permalink hash.
+     *
+     * @return array $filtered array containing permalink data.
+     */
+    private function filterSmallHash($smallHash)
+    {
+        $filtered = array();
+        foreach ($this->links as $l) {
+            if ($smallHash == smallHash($l['linkdate'])) {
+                // Yes, this is ugly and slow
+                $filtered[$l['linkdate']] = $l;
+                return $filtered;
+            }
+        }
+        return $filtered;
+    }
+
+    /**
+     * Returns the list of links corresponding to a full-text search
+     *
+     * Searches:
+     *  - in the URLs, title and description;
+     *  - are case-insensitive.
+     *
+     * Example:
+     *    print_r($mydb->filterFulltext('hollandais'));
+     *
+     * mb_convert_case($val, MB_CASE_LOWER, 'UTF-8')
+     *  - allows to perform searches on Unicode text
+     *  - see https://github.com/shaarli/Shaarli/issues/75 for examples
+     *
+     * @param string $searchterms search query.
+     * @param bool   $privateonly return only private links if true.
+     *
+     * @return array search results.
+     */
+    private function filterFulltext($searchterms, $privateonly = false)
+    {
+        // FIXME: explode(' ',$searchterms) and perform a AND search.
+        // FIXME: accept double-quotes to search for a string "as is"?
+        $filtered = array();
+        $search = mb_convert_case($searchterms, MB_CASE_LOWER, 'UTF-8');
+        $explodedSearch = explode(' ', trim($search));
+        $keys = array('title', 'description', 'url', 'tags');
+
+        // Iterate over every stored link.
+        foreach ($this->links as $link) {
+            $found = false;
+
+            // ignore non private links when 'privatonly' is on.
+            if (! $link['private'] && $privateonly === true) {
+                continue;
+            }
+
+            // Iterate over searchable link fields.
+            foreach ($keys as $key) {
+                // Search full expression.
+                if (strpos(
+                    mb_convert_case($link[$key], MB_CASE_LOWER, 'UTF-8'),
+                    $search
+                ) !== false) {
+                    $found = true;
+                }
+
+                if ($found) {
+                    break;
+                }
+            }
+
+            if ($found) {
+                $filtered[$link['linkdate']] = $link;
+            }
+        }
+
+        krsort($filtered);
+        return $filtered;
+    }
+
+    /**
+     * Returns the list of links associated with a given list of tags
+     *
+     * You can specify one or more tags, separated by space or a comma, e.g.
+     *  print_r($mydb->filterTags('linux programming'));
+     *
+     * @param string $tags          list of tags separated by commas or blank spaces.
+     * @param bool   $casesensitive ignore case if false.
+     * @param bool   $privateonly   returns private links only.
+     *
+     * @return array filtered links.
+     */
+    public function filterTags($tags, $casesensitive = false, $privateonly = false)
+    {
+        $searchtags = $this->tagsStrToArray($tags, $casesensitive);
+        $filtered = array();
+
+        foreach ($this->links as $l) {
+            // ignore non private links when 'privatonly' is on.
+            if (! $l['private'] && $privateonly === true) {
+                continue;
+            }
+
+            $linktags = $this->tagsStrToArray($l['tags'], $casesensitive);
+
+            if (count(array_intersect($linktags, $searchtags)) == count($searchtags)) {
+                $filtered[$l['linkdate']] = $l;
+            }
+        }
+        krsort($filtered);
+        return $filtered;
+    }
+
+    /**
+     * Returns the list of articles for a given day, chronologically sorted
+     *
+     * Day must be in the form 'YYYYMMDD' (e.g. '20120125'), e.g.
+     *  print_r($mydb->filterDay('20120125'));
+     *
+     * @param string $day day to filter.
+     *
+     * @return array all link matching given day.
+     *
+     * @throws Exception if date format is invalid.
+     */
+    public function filterDay($day)
+    {
+        if (! checkDateFormat('Ymd', $day)) {
+            throw new Exception('Invalid date format');
+        }
+
+        $filtered = array();
+        foreach ($this->links as $l) {
+            if (startsWith($l['linkdate'], $day)) {
+                $filtered[$l['linkdate']] = $l;
+            }
+        }
+        ksort($filtered);
+        return $filtered;
+    }
+
+    /**
+     * Convert a list of tags (str) to an array. Also
+     * - handle case sensitivity.
+     * - accepts spaces commas as separator.
+     * - remove private tags for loggedout users.
+     *
+     * @param string $tags          string containing a list of tags.
+     * @param bool   $casesensitive will convert everything to lowercase if false.
+     *
+     * @return array filtered tags string.
+    */
+    public function tagsStrToArray($tags, $casesensitive)
+    {
+        // We use UTF-8 conversion to handle various graphemes (i.e. cyrillic, or greek)
+        $tagsOut = $casesensitive ? $tags : mb_convert_case($tags, MB_CASE_LOWER, 'UTF-8');
+        $tagsOut = str_replace(',', ' ', $tagsOut);
+
+        return explode(' ', trim($tagsOut));
+    }
+}
diff --git a/application/LinkUtils.php b/application/LinkUtils.php
new file mode 100644
index 00000000..26dd6b67
--- /dev/null
+++ b/application/LinkUtils.php
@@ -0,0 +1,79 @@
+<?php
+
+/**
+ * Extract title from an HTML document.
+ *
+ * @param string $html HTML content where to look for a title.
+ *
+ * @return bool|string Extracted title if found, false otherwise.
+ */
+function html_extract_title($html)
+{
+    if (preg_match('!<title>(.*)</title>!is', $html, $matches)) {
+        return trim(str_replace("\n", ' ', $matches[1]));
+    }
+    return false;
+}
+
+/**
+ * Determine charset from downloaded page.
+ * Priority:
+ *   1. HTTP headers (Content type).
+ *   2. HTML content page (tag <meta charset>).
+ *   3. Use a default charset (default: UTF-8).
+ *
+ * @param array  $headers           HTTP headers array.
+ * @param string $htmlContent       HTML content where to look for charset.
+ * @param string $defaultCharset    Default charset to apply if other methods failed.
+ *
+ * @return string Determined charset.
+ */
+function get_charset($headers, $htmlContent, $defaultCharset = 'utf-8')
+{
+    if ($charset = headers_extract_charset($headers)) {
+        return $charset;
+    }
+
+    if ($charset = html_extract_charset($htmlContent)) {
+        return $charset;
+    }
+
+    return $defaultCharset;
+}
+
+/**
+ * Extract charset from HTTP headers if it's defined.
+ *
+ * @param array $headers HTTP headers array.
+ *
+ * @return bool|string Charset string if found (lowercase), false otherwise.
+ */
+function headers_extract_charset($headers)
+{
+    if (! empty($headers['Content-Type']) && strpos($headers['Content-Type'], 'charset=') !== false) {
+        preg_match('/charset="?([^; ]+)/i', $headers['Content-Type'], $match);
+        if (! empty($match[1])) {
+            return strtolower(trim($match[1]));
+        }
+    }
+
+    return false;
+}
+
+/**
+ * Extract charset HTML content (tag <meta charset>).
+ *
+ * @param string $html HTML content where to look for charset.
+ *
+ * @return bool|string Charset string if found, false otherwise.
+ */
+function html_extract_charset($html)
+{
+    // Get encoding specified in HTML header.
+    preg_match('#<meta .*charset="?([^">/]+)"? */?>#Usi', $html, $enc);
+    if (!empty($enc[1])) {
+        return strtolower($enc[1]);
+    }
+
+    return false;
+}
diff --git a/application/Url.php b/application/Url.php
index af43b457..a4ac2e73 100644
--- a/application/Url.php
+++ b/application/Url.php
@@ -52,6 +52,18 @@ function get_url_scheme($url)
 }
 
 /**
+ * Adds a trailing slash at the end of URL if necessary.
+ *
+ * @param string $url URL to check/edit.
+ *
+ * @return string $url URL with a end trailing slash.
+ */
+function add_trailing_slash($url)
+{
+    return $url . (!endsWith($url, '/') ? '/' : '');
+}
+
+/**
  * URL representation and cleanup utilities
  *
  * Form
@@ -106,7 +118,7 @@ class Url
      */
     public function __construct($url)
     {
-        $this->parts = parse_url($url);
+        $this->parts = parse_url(trim($url));
 
         if (!empty($url) && empty($this->parts['scheme'])) {
             $this->parts['scheme'] = 'http';
@@ -189,4 +201,13 @@ class Url
         }
         return $this->parts['scheme'];
     }
+
+    /**
+     * Test if the Url is an HTTP one.
+     *
+     * @return true is HTTP, false otherwise.
+     */
+    public function isHttp() {
+        return strpos(strtolower($this->parts['scheme']), 'http') !== false;
+    }
 }
diff --git a/application/Utils.php b/application/Utils.php
index ac8bfbfc..10d60698 100644
--- a/application/Utils.php
+++ b/application/Utils.php
@@ -4,6 +4,24 @@
  */
 
 /**
+ * Logs a message to a text file
+ *
+ * The log format is compatible with fail2ban.
+ *
+ * @param string $logFile  where to write the logs
+ * @param string $clientIp the client's remote IPv4/IPv6 address
+ * @param string $message  the message to log
+ */
+function logm($logFile, $clientIp, $message)
+{
+    file_put_contents(
+        $logFile,
+        date('Y/m/d H:i:s').' - '.$clientIp.' - '.strval($message).PHP_EOL,
+        FILE_APPEND
+    );
+}
+
+/**
  *  Returns the small hash of a string, using RFC 4648 base64url format
  *
  *  Small hashes:
@@ -64,12 +82,14 @@ function sanitizeLink(&$link)
 
 /**
  * Checks if a string represents a valid date
+
+ * @param string $format The expected DateTime format of the string
+ * @param string $string A string-formatted date
+ *
+ * @return bool whether the string is a valid date
  *
- * @param string        a string-formatted date
- * @param format        the expected DateTime format of the string
- * @return              whether the string is a valid date
- * @see                 http://php.net/manual/en/class.datetime.php
- * @see                 http://php.net/manual/en/datetime.createfromformat.php
+ * @see http://php.net/manual/en/class.datetime.php
+ * @see http://php.net/manual/en/datetime.createfromformat.php
  */
 function checkDateFormat($format, $string)
 {