application/bookmark/BookmarkFilter.php

   1 <?php
   2
   3 declare(strict_types=1);
   4
   5 namespace Shaarli\Bookmark;
   6
   7 use Exception;
   8 use Shaarli\Bookmark\Exception\BookmarkNotFoundException;
   9
  10 /**
  11  * Class LinkFilter.
  12  *
  13  * Perform search and filter operation on link data list.
  14  */
  15 class BookmarkFilter
  16 {
  17     /**
  18      * @var string permalinks.
  19      */
  20     public static $FILTER_HASH = 'permalink';
  21
  22     /**
  23      * @var string text search.
  24      */
  25     public static $FILTER_TEXT = 'fulltext';
  26
  27     /**
  28      * @var string tag filter.
  29      */
  30     public static $FILTER_TAG = 'tags';
  31
  32     /**
  33      * @var string filter by day.
  34      */
  35     public static $FILTER_DAY = 'FILTER_DAY';
  36
  37     /**
  38      * @var string filter by day.
  39      */
  40     public static $DEFAULT = 'NO_FILTER';
  41
  42     /** @var string Visibility: all */
  43     public static $ALL = 'all';
  44
  45     /** @var string Visibility: public */
  46     public static $PUBLIC = 'public';
  47
  48     /** @var string Visibility: private */
  49     public static $PRIVATE = 'private';
  50
  51     /**
  52      * @var string Allowed characters for hashtags (regex syntax).
  53      */
  54     public static $HASHTAG_CHARS = '\p{Pc}\p{N}\p{L}\p{Mn}';
  55
  56     /**
  57      * @var Bookmark[] all available bookmarks.
  58      */
  59     private $bookmarks;
  60
  61     /**
  62      * @param Bookmark[] $bookmarks initialization.
  63      */
  64     public function __construct($bookmarks)
  65     {
  66         $this->bookmarks = $bookmarks;
  67     }
  68
  69     /**
  70      * Filter bookmarks according to parameters.
  71      *
  72      * @param string $type          Type of filter (eg. tags, permalink, etc.).
  73      * @param mixed  $request       Filter content.
  74      * @param bool   $casesensitive Optional: Perform case sensitive filter if true.
  75      * @param string $visibility    Optional: return only all/private/public bookmarks
  76      * @param bool   $untaggedonly  Optional: return only untagged bookmarks. Applies only if $type includes FILTER_TAG
  77      *
  78      * @return Bookmark[] filtered bookmark list.
  79      *
  80      * @throws BookmarkNotFoundException
  81      */
  82     public function filter(
  83         string $type,
  84         $request,
  85         bool $casesensitive = false,
  86         string $visibility = 'all',
  87         bool $untaggedonly = false
  88     ) {
  89         if (!in_array($visibility, ['all', 'public', 'private'])) {
  90             $visibility = 'all';
  91         }
  92
  93         switch ($type) {
  94             case self::$FILTER_HASH:
  95                 return $this->filterSmallHash($request);
  96             case self::$FILTER_TAG | self::$FILTER_TEXT: // == "vuotext"
  97                 $noRequest = empty($request) || (empty($request[0]) && empty($request[1]));
  98                 if ($noRequest) {
  99                     if ($untaggedonly) {
 100                         return $this->filterUntagged($visibility);
 101                     }
 102                     return $this->noFilter($visibility);
 103                 }
 104                 if ($untaggedonly) {
 105                     $filtered = $this->filterUntagged($visibility);
 106                 } else {
 107                     $filtered = $this->bookmarks;
 108                 }
 109                 if (!empty($request[0])) {
 110                     $filtered = (new BookmarkFilter($filtered))->filterTags($request[0], $casesensitive, $visibility);
 111                 }
 112                 if (!empty($request[1])) {
 113                     $filtered = (new BookmarkFilter($filtered))->filterFulltext($request[1], $visibility);
 114                 }
 115                 return $filtered;
 116             case self::$FILTER_TEXT:
 117                 return $this->filterFulltext($request, $visibility);
 118             case self::$FILTER_TAG:
 119                 if ($untaggedonly) {
 120                     return $this->filterUntagged($visibility);
 121                 } else {
 122                     return $this->filterTags($request, $casesensitive, $visibility);
 123                 }
 124             case self::$FILTER_DAY:
 125                 return $this->filterDay($request, $visibility);
 126             default:
 127                 return $this->noFilter($visibility);
 128         }
 129     }
 130
 131     /**
 132      * Unknown filter, but handle private only.
 133      *
 134      * @param string $visibility Optional: return only all/private/public bookmarks
 135      *
 136      * @return Bookmark[] filtered bookmarks.
 137      */
 138     private function noFilter(string $visibility = 'all')
 139     {
 140         if ($visibility === 'all') {
 141             return $this->bookmarks;
 142         }
 143
 144         $out = array();
 145         foreach ($this->bookmarks as $key => $value) {
 146             if ($value->isPrivate() && $visibility === 'private') {
 147                 $out[$key] = $value;
 148             } elseif (!$value->isPrivate() && $visibility === 'public') {
 149                 $out[$key] = $value;
 150             }
 151         }
 152
 153         return $out;
 154     }
 155
 156     /**
 157      * Returns the shaare corresponding to a smallHash.
 158      *
 159      * @param string $smallHash permalink hash.
 160      *
 161      * @return Bookmark[] $filtered array containing permalink data.
 162      *
 163      * @throws BookmarkNotFoundException if the smallhash doesn't match any link.
 164      */
 165     private function filterSmallHash(string $smallHash)
 166     {
 167         foreach ($this->bookmarks as $key => $l) {
 168             if ($smallHash == $l->getShortUrl()) {
 169                 // Yes, this is ugly and slow
 170                 return [$key => $l];
 171             }
 172         }
 173
 174         throw new BookmarkNotFoundException();
 175     }
 176
 177     /**
 178      * Returns the list of bookmarks corresponding to a full-text search
 179      *
 180      * Searches:
 181      *  - in the URLs, title and description;
 182      *  - are case-insensitive;
 183      *  - terms surrounded by quotes " are exact terms search.
 184      *  - terms starting with a dash - are excluded (except exact terms).
 185      *
 186      * Example:
 187      *    print_r($mydb->filterFulltext('hollandais'));
 188      *
 189      * mb_convert_case($val, MB_CASE_LOWER, 'UTF-8')
 190      *  - allows to perform searches on Unicode text
 191      *  - see https://github.com/shaarli/Shaarli/issues/75 for examples
 192      *
 193      * @param string $searchterms search query.
 194      * @param string $visibility  Optional: return only all/private/public bookmarks.
 195      *
 196      * @return Bookmark[] search results.
 197      */
 198     private function filterFulltext(string $searchterms, string $visibility = 'all')
 199     {
 200         if (empty($searchterms)) {
 201             return $this->noFilter($visibility);
 202         }
 203
 204         $filtered = [];
 205         $search = mb_convert_case(html_entity_decode($searchterms), MB_CASE_LOWER, 'UTF-8');
 206         $exactRegex = '/"([^"]+)"/';
 207         // Retrieve exact search terms.
 208         preg_match_all($exactRegex, $search, $exactSearch);
 209         $exactSearch = array_values(array_filter($exactSearch[1]));
 210
 211         // Remove exact search terms to get AND terms search.
 212         $explodedSearchAnd = explode(' ', trim(preg_replace($exactRegex, '', $search)));
 213         $explodedSearchAnd = array_values(array_filter($explodedSearchAnd));
 214
 215         // Filter excluding terms and update andSearch.
 216         $excludeSearch = [];
 217         $andSearch = [];
 218         foreach ($explodedSearchAnd as $needle) {
 219             if ($needle[0] == '-' && strlen($needle) > 1) {
 220                 $excludeSearch[] = substr($needle, 1);
 221             } else {
 222                 $andSearch[] = $needle;
 223             }
 224         }
 225
 226         // Iterate over every stored link.
 227         foreach ($this->bookmarks as $id => $link) {
 228             // ignore non private bookmarks when 'privatonly' is on.
 229             if ($visibility !== 'all') {
 230                 if (!$link->isPrivate() && $visibility === 'private') {
 231                     continue;
 232                 } elseif ($link->isPrivate() && $visibility === 'public') {
 233                     continue;
 234                 }
 235             }
 236
 237             $lengths = [];
 238             $content = $this->buildFullTextSearchableLink($link, $lengths);
 239
 240             // Be optimistic
 241             $found = true;
 242             $foundPositions = [];
 243
 244             // First, we look for exact term search
 245             // Then iterate over keywords, if keyword is not found,
 246             // no need to check for the others. We want all or nothing.
 247             foreach ([$exactSearch, $andSearch] as $search) {
 248                 for ($i = 0; $i < count($search) && $found !== false; $i++) {
 249                     $found = mb_strpos($content, $search[$i]);
 250                     if ($found === false) {
 251                         break;
 252                     }
 253
 254                     $foundPositions[] = ['start' => $found, 'end' => $found + mb_strlen($search[$i])];
 255                 }
 256             }
 257
 258             // Exclude terms.
 259             for ($i = 0; $i < count($excludeSearch) && $found !== false; $i++) {
 260                 $found = strpos($content, $excludeSearch[$i]) === false;
 261             }
 262
 263             if ($found !== false) {
 264                 $link->addAdditionalContentEntry(
 265                     'search_highlight',
 266                     $this->postProcessFoundPositions($lengths, $foundPositions)
 267                 );
 268
 269                 $filtered[$id] = $link;
 270             }
 271         }
 272
 273         return $filtered;
 274     }
 275
 276     /**
 277      * generate a regex fragment out of a tag
 278      *
 279      * @param string $tag to to generate regexs from. may start with '-' to negate, contain '*' as wildcard
 280      *
 281      * @return string generated regex fragment
 282      */
 283     private static function tag2regex(string $tag): string
 284     {
 285         $len = strlen($tag);
 286         if (!$len || $tag === "-" || $tag === "*") {
 287             // nothing to search, return empty regex
 288             return '';
 289         }
 290         if ($tag[0] === "-") {
 291             // query is negated
 292             $i = 1; // use offset to start after '-' character
 293             $regex = '(?!'; // create negative lookahead
 294         } else {
 295             $i = 0; // start at first character
 296             $regex = '(?='; // use positive lookahead
 297         }
 298         $regex .= '.*(?:^| )'; // before tag may only be a space or the beginning
 299         // iterate over string, separating it into placeholder and content
 300         for (; $i < $len; $i++) {
 301             if ($tag[$i] === '*') {
 302                 // placeholder found
 303                 $regex .= '[^ ]*?';
 304             } else {
 305                 // regular characters
 306                 $offset = strpos($tag, '*', $i);
 307                 if ($offset === false) {
 308                     // no placeholder found, set offset to end of string
 309                     $offset = $len;
 310                 }
 311                 // subtract one, as we want to get before the placeholder or end of string
 312                 $offset -= 1;
 313                 // we got a tag name that we want to search for. escape any regex characters to prevent conflicts.
 314                 $regex .= preg_quote(substr($tag, $i, $offset - $i + 1), '/');
 315                 // move $i on
 316                 $i = $offset;
 317             }
 318         }
 319         $regex .= '(?:$| ))'; // after the tag may only be a space or the end
 320         return $regex;
 321     }
 322
 323     /**
 324      * Returns the list of bookmarks associated with a given list of tags
 325      *
 326      * You can specify one or more tags, separated by space or a comma, e.g.
 327      *  print_r($mydb->filterTags('linux programming'));
 328      *
 329      * @param string|array $tags          list of tags, separated by commas or blank spaces if passed as string.
 330      * @param bool         $casesensitive ignore case if false.
 331      * @param string       $visibility    Optional: return only all/private/public bookmarks.
 332      *
 333      * @return Bookmark[] filtered bookmarks.
 334      */
 335     public function filterTags($tags, bool $casesensitive = false, string $visibility = 'all')
 336     {
 337         // get single tags (we may get passed an array, even though the docs say different)
 338         $inputTags = $tags;
 339         if (!is_array($tags)) {
 340             // we got an input string, split tags
 341             $inputTags = preg_split('/(?:\s+)|,/', $inputTags, -1, PREG_SPLIT_NO_EMPTY);
 342         }
 343
 344         if (!count($inputTags)) {
 345             // no input tags
 346             return $this->noFilter($visibility);
 347         }
 348
 349         // If we only have public visibility, we can't look for hidden tags
 350         if ($visibility === self::$PUBLIC) {
 351             $inputTags = array_values(array_filter($inputTags, function ($tag) {
 352                 return ! startsWith($tag, '.');
 353             }));
 354
 355             if (empty($inputTags)) {
 356                 return [];
 357             }
 358         }
 359
 360         // build regex from all tags
 361         $re = '/^' . implode(array_map("self::tag2regex", $inputTags)) . '.*$/';
 362         if (!$casesensitive) {
 363             // make regex case insensitive
 364             $re .= 'i';
 365         }
 366
 367         // create resulting array
 368         $filtered = [];
 369
 370         // iterate over each link
 371         foreach ($this->bookmarks as $key => $link) {
 372             // check level of visibility
 373             // ignore non private bookmarks when 'privateonly' is on.
 374             if ($visibility !== 'all') {
 375                 if (!$link->isPrivate() && $visibility === 'private') {
 376                     continue;
 377                 } elseif ($link->isPrivate() && $visibility === 'public') {
 378                     continue;
 379                 }
 380             }
 381             $search = $link->getTagsString(); // build search string, start with tags of current link
 382             if (strlen(trim($link->getDescription())) && strpos($link->getDescription(), '#') !== false) {
 383                 // description given and at least one possible tag found
 384                 $descTags = array();
 385                 // find all tags in the form of #tag in the description
 386                 preg_match_all(
 387                     '/(?<![' . self::$HASHTAG_CHARS . '])#([' . self::$HASHTAG_CHARS . ']+?)\b/sm',
 388                     $link->getDescription(),
 389                     $descTags
 390                 );
 391                 if (count($descTags[1])) {
 392                     // there were some tags in the description, add them to the search string
 393                     $search .= ' ' . implode(' ', $descTags[1]);
 394                 }
 395             };
 396             // match regular expression with search string
 397             if (!preg_match($re, $search)) {
 398                 // this entry does _not_ match our regex
 399                 continue;
 400             }
 401             $filtered[$key] = $link;
 402         }
 403         return $filtered;
 404     }
 405
 406     /**
 407      * Return only bookmarks without any tag.
 408      *
 409      * @param string $visibility return only all/private/public bookmarks.
 410      *
 411      * @return Bookmark[] filtered bookmarks.
 412      */
 413     public function filterUntagged(string $visibility)
 414     {
 415         $filtered = [];
 416         foreach ($this->bookmarks as $key => $link) {
 417             if ($visibility !== 'all') {
 418                 if (!$link->isPrivate() && $visibility === 'private') {
 419                     continue;
 420                 } elseif ($link->isPrivate() && $visibility === 'public') {
 421                     continue;
 422                 }
 423             }
 424
 425             if (empty(trim($link->getTagsString()))) {
 426                 $filtered[$key] = $link;
 427             }
 428         }
 429
 430         return $filtered;
 431     }
 432
 433     /**
 434      * Returns the list of articles for a given day, chronologically sorted
 435      *
 436      * Day must be in the form 'YYYYMMDD' (e.g. '20120125'), e.g.
 437      *  print_r($mydb->filterDay('20120125'));
 438      *
 439      * @param string $day day to filter.
 440      * @param string $visibility return only all/private/public bookmarks.
 441
 442      * @return Bookmark[] all link matching given day.
 443      *
 444      * @throws Exception if date format is invalid.
 445      */
 446     public function filterDay(string $day, string $visibility)
 447     {
 448         if (!checkDateFormat('Ymd', $day)) {
 449             throw new Exception('Invalid date format');
 450         }
 451
 452         $filtered = [];
 453         foreach ($this->bookmarks as $key => $bookmark) {
 454             if ($visibility === static::$PUBLIC && $bookmark->isPrivate()) {
 455                 continue;
 456             }
 457
 458             if ($bookmark->getCreated()->format('Ymd') == $day) {
 459                 $filtered[$key] = $bookmark;
 460             }
 461         }
 462
 463         // sort by date ASC
 464         return array_reverse($filtered, true);
 465     }
 466
 467     /**
 468      * Convert a list of tags (str) to an array. Also
 469      * - handle case sensitivity.
 470      * - accepts spaces commas as separator.
 471      *
 472      * @param string $tags          string containing a list of tags.
 473      * @param bool   $casesensitive will convert everything to lowercase if false.
 474      *
 475      * @return string[] filtered tags string.
 476      */
 477     public static function tagsStrToArray(string $tags, bool $casesensitive): array
 478     {
 479         // We use UTF-8 conversion to handle various graphemes (i.e. cyrillic, or greek)
 480         $tagsOut = $casesensitive ? $tags : mb_convert_case($tags, MB_CASE_LOWER, 'UTF-8');
 481         $tagsOut = str_replace(',', ' ', $tagsOut);
 482
 483         return preg_split('/\s+/', $tagsOut, -1, PREG_SPLIT_NO_EMPTY);
 484     }
 485
 486     /**
 487      * This method finalize the content of the foundPositions array,
 488      * by associated all search results to their associated bookmark field,
 489      * making sure that there is no overlapping results, etc.
 490      *
 491      * @param array $fieldLengths   Start and end positions of every bookmark fields in the aggregated bookmark content.
 492      * @param array $foundPositions Positions where the search results were found in the aggregated content.
 493      *
 494      * @return array Updated $foundPositions, by bookmark field.
 495      */
 496     protected function postProcessFoundPositions(array $fieldLengths, array $foundPositions): array
 497     {
 498         // Sort results by starting position ASC.
 499         usort($foundPositions, function (array $entryA, array $entryB): int {
 500             return $entryA['start'] > $entryB['start'] ? 1 : -1;
 501         });
 502
 503         $out = [];
 504         $currentMax = -1;
 505         foreach ($foundPositions as $foundPosition) {
 506             // we do not allow overlapping highlights
 507             if ($foundPosition['start'] < $currentMax) {
 508                 continue;
 509             }
 510
 511             $currentMax = $foundPosition['end'];
 512             foreach ($fieldLengths as $part => $length) {
 513                 if ($foundPosition['start'] < $length['start'] || $foundPosition['start'] > $length['end']) {
 514                     continue;
 515                 }
 516
 517                 $out[$part][] = [
 518                     'start' => $foundPosition['start'] - $length['start'],
 519                     'end' => $foundPosition['end'] - $length['start'],
 520                 ];
 521                 break;
 522             }
 523         }
 524
 525         return $out;
 526     }
 527
 528     /**
 529      * Concatenate link fields to search across fields. Adds a '\' separator for exact search terms.
 530      * Also populate $length array with starting and ending positions of every bookmark field
 531      * inside concatenated content.
 532      *
 533      * @param Bookmark $link
 534      * @param array    $lengths (by reference)
 535      *
 536      * @return string Lowercase concatenated fields content.
 537      */
 538     protected function buildFullTextSearchableLink(Bookmark $link, array &$lengths): string
 539     {
 540         $content  = mb_convert_case($link->getTitle(), MB_CASE_LOWER, 'UTF-8') .'\\';
 541         $content .= mb_convert_case($link->getDescription(), MB_CASE_LOWER, 'UTF-8') .'\\';
 542         $content .= mb_convert_case($link->getUrl(), MB_CASE_LOWER, 'UTF-8') .'\\';
 543         $content .= mb_convert_case($link->getTagsString(), MB_CASE_LOWER, 'UTF-8') .'\\';
 544
 545         $lengths['title'] = ['start' => 0, 'end' => mb_strlen($link->getTitle())];
 546         $nextField = $lengths['title']['end'] + 1;
 547         $lengths['description'] = ['start' => $nextField, 'end' => $nextField + mb_strlen($link->getDescription())];
 548         $nextField = $lengths['description']['end'] + 1;
 549         $lengths['url'] = ['start' => $nextField, 'end' => $nextField + mb_strlen($link->getUrl())];
 550         $nextField = $lengths['url']['end'] + 1;
 551         $lengths['tags'] = ['start' => $nextField, 'end' => $nextField + mb_strlen($link->getTagsString())];
 552
 553         return $content;
 554     }
 555 }