From fe3713d2e5c91e2d07af72b39f321521d3dd470c Mon Sep 17 00:00:00 2001 From: VirtualTam Date: Mon, 3 Dec 2018 01:35:14 +0100 Subject: namespacing: move LinkUtils along \Shaarli\Bookmark classes Signed-off-by: VirtualTam --- application/bookmark/LinkUtils.php | 222 +++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 application/bookmark/LinkUtils.php (limited to 'application/bookmark/LinkUtils.php') diff --git a/application/bookmark/LinkUtils.php b/application/bookmark/LinkUtils.php new file mode 100644 index 00000000..de5b61cb --- /dev/null +++ b/application/bookmark/LinkUtils.php @@ -0,0 +1,222 @@ +(.*?)!is', $html, $matches)) { + return trim(str_replace("\n", '', $matches[1])); + } + return false; +} + +/** + * Extract charset from HTTP header if it's defined. + * + * @param string $header HTTP header Content-Type line. + * + * @return bool|string Charset string if found (lowercase), false otherwise. + */ +function header_extract_charset($header) +{ + preg_match('/charset="?([^; ]+)/i', $header, $match); + if (! empty($match[1])) { + return strtolower(trim($match[1])); + } + + return false; +} + +/** + * Extract charset HTML content (tag ). + * + * @param string $html HTML content where to look for charset. + * + * @return bool|string Charset string if found, false otherwise. + */ +function html_extract_charset($html) +{ + // Get encoding specified in HTML header. + preg_match('#/]+)["\']? */?>#Usi', $html, $enc); + if (!empty($enc[1])) { + return strtolower($enc[1]); + } + + return false; +} + +/** + * Count private links in given linklist. + * + * @param array|Countable $links Linklist. + * + * @return int Number of private links. + */ +function count_private($links) +{ + $cpt = 0; + foreach ($links as $link) { + if ($link['private']) { + $cpt += 1; + } + } + + return $cpt; +} + +/** + * In a string, converts URLs to clickable links. + * + * @param string $text input string. + * @param string $redirector if a redirector is set, use it to gerenate links. + * @param bool $urlEncode Use `urlencode()` on the URL after the redirector or not. + * + * @return string returns $text with all links converted to HTML links. + * + * @see Function inspired from http://www.php.net/manual/en/function.preg-replace.php#85722 + */ +function text2clickable($text, $redirector = '', $urlEncode = true) +{ + $regex = '!(((?:https?|ftp|file)://|apt:|magnet:)\S+[a-z0-9\(\)]/?)!si'; + + if (empty($redirector)) { + return preg_replace($regex, '$1', $text); + } + // Redirector is set, urlencode the final URL. + return preg_replace_callback( + $regex, + function ($matches) use ($redirector, $urlEncode) { + $url = $urlEncode ? urlencode($matches[1]) : $matches[1]; + return ''. $matches[1] .''; + }, + $text + ); +} + +/** + * Auto-link hashtags. + * + * @param string $description Given description. + * @param string $indexUrl Root URL. + * + * @return string Description with auto-linked hashtags. + */ +function hashtag_autolink($description, $indexUrl = '') +{ + /* + * To support unicode: http://stackoverflow.com/a/35498078/1484919 + * \p{Pc} - to match underscore + * \p{N} - numeric character in any script + * \p{L} - letter from any language + * \p{Mn} - any non marking space (accents, umlauts, etc) + */ + $regex = '/(^|\s)#([\p{Pc}\p{N}\p{L}\p{Mn}]+)/mui'; + $replacement = '$1#$2'; + return preg_replace($regex, $replacement, $description); +} + +/** + * This function inserts   where relevant so that multiple spaces are properly displayed in HTML + * even in the absence of 
  (This is used in description to keep text formatting).
+ *
+ * @param string $text input text.
+ *
+ * @return string formatted text.
+ */
+function space2nbsp($text)
+{
+    return preg_replace('/(^| ) /m', '$1 ', $text);
+}
+
+/**
+ * Format Shaarli's description
+ *
+ * @param string $description shaare's description.
+ * @param string $redirector  if a redirector is set, use it to gerenate links.
+ * @param bool   $urlEncode   Use `urlencode()` on the URL after the redirector or not.
+ * @param string $indexUrl    URL to Shaarli's index.
+
+ * @return string formatted description.
+ */
+function format_description($description, $redirector = '', $urlEncode = true, $indexUrl = '')
+{
+    return nl2br(space2nbsp(hashtag_autolink(text2clickable($description, $redirector, $urlEncode), $indexUrl)));
+}
+
+/**
+ * Generate a small hash for a link.
+ *
+ * @param DateTime $date Link creation date.
+ * @param int      $id   Link ID.
+ *
+ * @return string the small hash generated from link data.
+ */
+function link_small_hash($date, $id)
+{
+    return smallHash($date->format(LinkDB::LINK_DATE_FORMAT) . $id);
+}
-- 
cgit v1.2.3


From 520d29578c57e476ece3bdd20c286d196b7b61b4 Mon Sep 17 00:00:00 2001
From: ArthurHoaro 
Date: Sat, 9 Feb 2019 13:52:12 +0100
Subject: Remove the redirector setting

Fixes #1239
---
 application/bookmark/LinkUtils.php | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)

(limited to 'application/bookmark/LinkUtils.php')

diff --git a/application/bookmark/LinkUtils.php b/application/bookmark/LinkUtils.php
index de5b61cb..988970bd 100644
--- a/application/bookmark/LinkUtils.php
+++ b/application/bookmark/LinkUtils.php
@@ -133,29 +133,15 @@ function count_private($links)
  * In a string, converts URLs to clickable links.
  *
  * @param string $text       input string.
- * @param string $redirector if a redirector is set, use it to gerenate links.
- * @param bool   $urlEncode  Use `urlencode()` on the URL after the redirector or not.
  *
  * @return string returns $text with all links converted to HTML links.
  *
  * @see Function inspired from http://www.php.net/manual/en/function.preg-replace.php#85722
  */
-function text2clickable($text, $redirector = '', $urlEncode = true)
+function text2clickable($text)
 {
     $regex = '!(((?:https?|ftp|file)://|apt:|magnet:)\S+[a-z0-9\(\)]/?)!si';
-
-    if (empty($redirector)) {
-        return preg_replace($regex, '$1', $text);
-    }
-    // Redirector is set, urlencode the final URL.
-    return preg_replace_callback(
-        $regex,
-        function ($matches) use ($redirector, $urlEncode) {
-            $url = $urlEncode ? urlencode($matches[1]) : $matches[1];
-            return ''. $matches[1] .'';
-        },
-        $text
-    );
+    return preg_replace($regex, '$1', $text);
 }
 
 /**
@@ -197,15 +183,13 @@ function space2nbsp($text)
  * Format Shaarli's description
  *
  * @param string $description shaare's description.
- * @param string $redirector  if a redirector is set, use it to gerenate links.
- * @param bool   $urlEncode   Use `urlencode()` on the URL after the redirector or not.
  * @param string $indexUrl    URL to Shaarli's index.
 
  * @return string formatted description.
  */
-function format_description($description, $redirector = '', $urlEncode = true, $indexUrl = '')
+function format_description($description, $indexUrl = '')
 {
-    return nl2br(space2nbsp(hashtag_autolink(text2clickable($description, $redirector, $urlEncode), $indexUrl)));
+    return nl2br(space2nbsp(hashtag_autolink(text2clickable($description), $indexUrl)));
 }
 
 /**
-- 
cgit v1.2.3


From a8e7da01146455f13ef06b151a7dafedd3acf769 Mon Sep 17 00:00:00 2001
From: ArthurHoaro 
Date: Sat, 9 Feb 2019 14:13:08 +0100
Subject: Do not try to retrieve thumbnails for internal link

Also adds a helper function to determine if a link is a note and apply it across multiple files.
---
 application/bookmark/LinkUtils.php | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'application/bookmark/LinkUtils.php')

diff --git a/application/bookmark/LinkUtils.php b/application/bookmark/LinkUtils.php
index de5b61cb..9e9d4f0a 100644
--- a/application/bookmark/LinkUtils.php
+++ b/application/bookmark/LinkUtils.php
@@ -220,3 +220,16 @@ function link_small_hash($date, $id)
 {
     return smallHash($date->format(LinkDB::LINK_DATE_FORMAT) . $id);
 }
+
+/**
+ * Returns whether or not the link is an internal note.
+ * Its URL starts by `?` because it's actually a permalink.
+ *
+ * @param string $linkUrl
+ *
+ * @return bool true if internal note, false otherwise.
+ */
+function is_note($linkUrl)
+{
+    return isset($linkUrl[0]) && $linkUrl[0] === '?';
+}
-- 
cgit v1.2.3


From 6a4872520cbbc012b5a8358cd50c78844afe8d07 Mon Sep 17 00:00:00 2001
From: ArthurHoaro 
Date: Sat, 8 Jun 2019 13:59:19 +0200
Subject: Automatically retrieve description for new bookmarks

If the option is enabled, it will try to find a meta tag containing
the page description and keywords, just like we do for the page title.
It will either look for regular meta tag or OpenGraph ones.

The option is disabled by default.

Note that keywords meta tags is mostly not used.

In `configure` template, the variable associated with this setting
is `$retrieve_description`.

Fixes #1302
---
 application/bookmark/LinkUtils.php | 85 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 81 insertions(+), 4 deletions(-)

(limited to 'application/bookmark/LinkUtils.php')

diff --git a/application/bookmark/LinkUtils.php b/application/bookmark/LinkUtils.php
index 35a5b290..77eb2d95 100644
--- a/application/bookmark/LinkUtils.php
+++ b/application/bookmark/LinkUtils.php
@@ -7,13 +7,25 @@ use Shaarli\Bookmark\LinkDB;
  *
  * @param string $charset     to extract from the downloaded page (reference)
  * @param string $title       to extract from the downloaded page (reference)
+ * @param string $description to extract from the downloaded page (reference)
+ * @param string $keywords    to extract from the downloaded page (reference)
+ * @param bool   $retrieveDescription Automatically tries to retrieve description and keywords from HTML content
  * @param string $curlGetInfo Optionally overrides curl_getinfo function
  *
  * @return Closure
  */
-function get_curl_download_callback(&$charset, &$title, $curlGetInfo = 'curl_getinfo')
-{
+function get_curl_download_callback(
+    &$charset,
+    &$title,
+    &$description,
+    &$keywords,
+    $retrieveDescription,
+    $curlGetInfo = 'curl_getinfo'
+) {
     $isRedirected = false;
+    $currentChunk = 0;
+    $foundChunk = null;
+
     /**
      * cURL callback function for CURLOPT_WRITEFUNCTION (called during the download).
      *
@@ -25,7 +37,18 @@ function get_curl_download_callback(&$charset, &$title, $curlGetInfo = 'curl_get
      *
      * @return int|bool length of $data or false if we need to stop the download
      */
-    return function (&$ch, $data) use ($curlGetInfo, &$charset, &$title, &$isRedirected) {
+    return function (&$ch, $data) use (
+        $retrieveDescription,
+        $curlGetInfo,
+        &$charset,
+        &$title,
+        &$description,
+        &$keywords,
+        &$isRedirected,
+        &$currentChunk,
+        &$foundChunk
+    ) {
+        $currentChunk++;
         $responseCode = $curlGetInfo($ch, CURLINFO_RESPONSE_CODE);
         if (!empty($responseCode) && in_array($responseCode, [301, 302])) {
             $isRedirected = true;
@@ -50,9 +73,34 @@ function get_curl_download_callback(&$charset, &$title, $curlGetInfo = 'curl_get
         }
         if (empty($title)) {
             $title = html_extract_title($data);
+            $foundChunk = ! empty($title) ? $currentChunk : $foundChunk;
+        }
+        if ($retrieveDescription && empty($description)) {
+            $description = html_extract_tag('description', $data);
+            $foundChunk = ! empty($description) ? $currentChunk : $foundChunk;
         }
+        if ($retrieveDescription && empty($keywords)) {
+            $keywords = html_extract_tag('keywords', $data);
+            if (! empty($keywords)) {
+                $foundChunk = $currentChunk;
+                // Keywords use the format tag1, tag2 multiple words, tag
+                // So we format them to match Shaarli's separator and glue multiple words with '-'
+                $keywords = implode(' ', array_map(function($keyword) {
+                    return implode('-', preg_split('/\s+/', trim($keyword)));
+                }, explode(',', $keywords)));
+            }
+        }
+
         // We got everything we want, stop the download.
-        if (!empty($responseCode) && !empty($contentType) && !empty($charset) && !empty($title)) {
+        // If we already found either the title, description or keywords,
+        // it's highly unlikely that we'll found the other metas further than
+        // in the same chunk of data or the next one. So we also stop the download after that.
+        if ((!empty($responseCode) && !empty($contentType) && !empty($charset)) && $foundChunk !== null
+            && (! $retrieveDescription
+                || $foundChunk < $currentChunk
+                || (!empty($title) && !empty($description) && !empty($keywords))
+            )
+        ) {
             return false;
         }
 
@@ -110,6 +158,35 @@ function html_extract_charset($html)
     return false;
 }
 
+/**
+ * Extract meta tag from HTML content in either:
+ *   - OpenGraph: 
+ *   - Meta tag: 
+ *
+ * @param string $tag  Name of the tag to retrieve.
+ * @param string $html HTML content where to look for charset.
+ *
+ * @return bool|string Charset string if found, false otherwise.
+ */
+function html_extract_tag($tag, $html)
+{
+    $propertiesKey = ['property', 'name', 'itemprop'];
+    $properties = implode('|', $propertiesKey);
+    // Try to retrieve OpenGraph image.
+    $ogRegex = '#]+(?:'. $properties .')=["\']?(?:og:)?'. $tag .'["\'\s][^>]*content=["\']?(.*?)["\'/>]#';
+    // If the attributes are not in the order property => content (e.g. Github)
+    // New regex to keep this readable... more or less.
+    $ogRegexReverse = '#]+content=["\']([^"\']+)[^>]+(?:'. $properties .')=["\']?(?:og)?:'. $tag .'["\'\s/>]#';
+
+    if (preg_match($ogRegex, $html, $matches) > 0
+        || preg_match($ogRegexReverse, $html, $matches) > 0
+    ) {
+        return $matches[1];
+    }
+
+    return false;
+}
+
 /**
  * Count private links in given linklist.
  *
-- 
cgit v1.2.3