Merge pull request #1567 from ArthurHoaro/feature/async-title-retrieval

8 files changed, 209 insertions, 80 deletions

diff --git a/application/config/ConfigManager.php b/application/config/ConfigManager.php
index 4c98be30..fb085023 100644
--- a/application/config/ConfigManager.php
+++ b/application/config/ConfigManager.php
@@ -366,7 +366,8 @@ class ConfigManager
         $this->setEmpty('general.links_per_page', 20);
         $this->setEmpty('general.enabled_plugins', self::$DEFAULT_PLUGINS);
         $this->setEmpty('general.default_note_title', 'Note: ');
-        $this->setEmpty('general.retrieve_description', false);
+        $this->setEmpty('general.retrieve_description', true);
+        $this->setEmpty('general.enable_async_metadata', true);
 
         $this->setEmpty('updates.check_updates', false);
         $this->setEmpty('updates.check_updates_branch', 'stable');
diff --git a/application/container/ContainerBuilder.php b/application/container/ContainerBuilder.php
index c21d58dd..fd94a1c3 100644
--- a/application/container/ContainerBuilder.php
+++ b/application/container/ContainerBuilder.php
@@ -14,6 +14,7 @@ use Shaarli\Front\Controller\Visitor\ErrorController;
 use Shaarli\Front\Controller\Visitor\ErrorNotFoundController;
 use Shaarli\History;
 use Shaarli\Http\HttpAccess;
+use Shaarli\Http\MetadataRetriever;
 use Shaarli\Netscape\NetscapeBookmarkUtils;
 use Shaarli\Plugin\PluginManager;
 use Shaarli\Render\PageBuilder;
@@ -90,6 +91,10 @@ class ContainerBuilder
             );
         };
 
+        $container['metadataRetriever'] = function (ShaarliContainer $container): MetadataRetriever {
+            return new MetadataRetriever($container->conf, $container->httpAccess);
+        };
+
         $container['pageBuilder'] = function (ShaarliContainer $container): PageBuilder {
             return new PageBuilder(
                 $container->conf,
diff --git a/application/container/ShaarliContainer.php b/application/container/ShaarliContainer.php
index 66e669aa..3a7c238f 100644
--- a/application/container/ShaarliContainer.php
+++ b/application/container/ShaarliContainer.php
@@ -10,6 +10,7 @@ use Shaarli\Feed\FeedBuilder;
 use Shaarli\Formatter\FormatterFactory;
 use Shaarli\History;
 use Shaarli\Http\HttpAccess;
+use Shaarli\Http\MetadataRetriever;
 use Shaarli\Netscape\NetscapeBookmarkUtils;
 use Shaarli\Plugin\PluginManager;
 use Shaarli\Render\PageBuilder;
@@ -35,6 +36,7 @@ use Slim\Container;
  * @property History                  $history
  * @property HttpAccess               $httpAccess
  * @property LoginManager             $loginManager
+ * @property MetadataRetriever        $metadataRetriever
  * @property NetscapeBookmarkUtils    $netscapeBookmarkUtils
  * @property callable                 $notFoundHandler       Overrides default Slim exception display
  * @property PageBuilder              $pageBuilder
diff --git a/application/front/controller/admin/ManageShaareController.php b/application/front/controller/admin/ManageShaareController.php
index bb083486..df2f1631 100644
--- a/application/front/controller/admin/ManageShaareController.php
+++ b/application/front/controller/admin/ManageShaareController.php
@@ -53,36 +53,22 @@ class ManageShaareController extends ShaarliAdminController
 
             // If this is an HTTP(S) link, we try go get the page to extract
             // the title (otherwise we will to straight to the edit form.)
-            if (empty($title) && strpos(get_url_scheme($url) ?: '', 'http') !== false) {
-                $retrieveDescription = $this->container->conf->get('general.retrieve_description');
-                // Short timeout to keep the application responsive
-                // The callback will fill $charset and $title with data from the downloaded page.
-                $this->container->httpAccess->getHttpResponse(
-                    $url,
-                    $this->container->conf->get('general.download_timeout', 30),
-                    $this->container->conf->get('general.download_max_size', 4194304),
-                    $this->container->httpAccess->getCurlDownloadCallback(
-                        $charset,
-                        $title,
-                        $description,
-                        $tags,
-                        $retrieveDescription
-                    )
-                );
-                if (! empty($title) && strtolower($charset) !== 'utf-8' && mb_check_encoding($charset)) {
-                    $title = mb_convert_encoding($title, 'utf-8', $charset);
-                }
+            if (true !== $this->container->conf->get('general.enable_async_metadata', true)
+                && empty($title)
+                && strpos(get_url_scheme($url) ?: '', 'http') !== false
+            ) {
+                $metadata = $this->container->metadataRetriever->retrieve($url);
             }
 
-            if (empty($url) && empty($title)) {
-                $title = $this->container->conf->get('general.default_note_title', t('Note: '));
+            if (empty($url)) {
+                $metadata['title'] = $this->container->conf->get('general.default_note_title', t('Note: '));
             }
 
             $link = [
-                'title' => $title,
+                'title' => $title ?? $metadata['title'] ?? '',
                 'url' => $url ?? '',
-                'description' => $description ?? '',
-                'tags' => $tags ?? '',
+                'description' => $description ?? $metadata['description'] ?? '',
+                'tags' => $tags ?? $metadata['tags'] ?? '',
                 'private' => $private,
             ];
         } else {
@@ -352,6 +338,8 @@ class ManageShaareController extends ShaarliAdminController
             'source' => $request->getParam('source') ?? '',
             'tags' => $tags,
             'default_private_links' => $this->container->conf->get('privacy.default_private_links', false),
+            'async_metadata' => $this->container->conf->get('general.enable_async_metadata', true),
+            'retrieve_description' => $this->container->conf->get('general.retrieve_description', false),
         ]);
 
         $this->executePageHooks('render_editlink', $data, TemplatePage::EDIT_LINK);
diff --git a/application/front/controller/admin/MetadataController.php b/application/front/controller/admin/MetadataController.php
new file mode 100644
index 00000000..ff845944
--- /dev/null
+++ b/application/front/controller/admin/MetadataController.php
@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shaarli\Front\Controller\Admin;
+
+use Slim\Http\Request;
+use Slim\Http\Response;
+
+/**
+ * Controller used to retrieve/update bookmark's metadata.
+ */
+class MetadataController extends ShaarliAdminController
+{
+    /**
+     * GET /admin/metadata/{url} - Attempt to retrieve the bookmark title from provided URL.
+     */
+    public function ajaxRetrieveTitle(Request $request, Response $response): Response
+    {
+        $url = $request->getParam('url');
+
+        // Only try to extract metadata from URL with HTTP(s) scheme
+        if (!empty($url) && strpos(get_url_scheme($url) ?: '', 'http') !== false) {
+            return $response->withJson($this->container->metadataRetriever->retrieve($url));
+        }
+
+        return $response->withJson([]);
+    }
+}
diff --git a/application/http/HttpAccess.php b/application/http/HttpAccess.php
index 81d9e076..646a5264 100644
--- a/application/http/HttpAccess.php
+++ b/application/http/HttpAccess.php
@@ -14,9 +14,14 @@ namespace Shaarli\Http;
  */
 class HttpAccess
 {
-    public function getHttpResponse($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null)
-    {
-        return get_http_response($url, $timeout, $maxBytes, $curlWriteFunction);
+    public function getHttpResponse(
+        $url,
+        $timeout = 30,
+        $maxBytes = 4194304,
+        $curlHeaderFunction = null,
+        $curlWriteFunction = null
+    ) {
+        return get_http_response($url, $timeout, $maxBytes, $curlHeaderFunction, $curlWriteFunction);
     }
 
     public function getCurlDownloadCallback(
@@ -24,16 +29,19 @@ class HttpAccess
         &$title,
         &$description,
         &$keywords,
-        $retrieveDescription,
-        $curlGetInfo = 'curl_getinfo'
+        $retrieveDescription
     ) {
         return get_curl_download_callback(
             $charset,
             $title,
             $description,
             $keywords,
-            $retrieveDescription,
-            $curlGetInfo
+            $retrieveDescription
         );
     }
+
+    public function getCurlHeaderCallback(&$charset, $curlGetInfo = 'curl_getinfo')
+    {
+        return get_curl_header_callback($charset, $curlGetInfo);
+    }
 }
diff --git a/application/http/HttpUtils.php b/application/http/HttpUtils.php
index 9f414073..28c12969 100644
--- a/application/http/HttpUtils.php
+++ b/application/http/HttpUtils.php
@@ -6,12 +6,14 @@ use Shaarli\Http\Url;
  * GET an HTTP URL to retrieve its content
  * Uses the cURL library or a fallback method
  *
- * @param string          $url               URL to get (http://...)
- * @param int             $timeout           network timeout (in seconds)
- * @param int             $maxBytes          maximum downloaded bytes (default: 4 MiB)
- * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
- *                                           Can be used to add download conditions on the
- *                                           headers (response code, content type, etc.).
+ * @param string          $url                URL to get (http://...)
+ * @param int             $timeout            network timeout (in seconds)
+ * @param int             $maxBytes           maximum downloaded bytes (default: 4 MiB)
+ * @param callable|string $curlHeaderFunction Optional callback called during the download of headers
+ *                                            (CURLOPT_HEADERFUNCTION)
+ * @param callable|string $curlWriteFunction  Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
+ *                                            Can be used to add download conditions on the
+ *                                            headers (response code, content type, etc.).
  *
  * @return array HTTP response headers, downloaded content
  *
@@ -35,8 +37,13 @@ use Shaarli\Http\Url;
  * @see http://stackoverflow.com/q/9183178
  * @see http://stackoverflow.com/q/1462720
  */
-function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null)
-{
+function get_http_response(
+    $url,
+    $timeout = 30,
+    $maxBytes = 4194304,
+    $curlHeaderFunction = null,
+    $curlWriteFunction = null
+) {
     $urlObj = new Url($url);
     $cleanUrl = $urlObj->idnToAscii();
 
@@ -70,7 +77,8 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteF
     // General cURL settings
     curl_setopt($ch, CURLOPT_AUTOREFERER, true);
     curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
-    curl_setopt($ch, CURLOPT_HEADER, true);
+    // Default header download if the $curlHeaderFunction is not defined
+    curl_setopt($ch, CURLOPT_HEADER, !is_callable($curlHeaderFunction));
     curl_setopt(
         $ch,
         CURLOPT_HTTPHEADER,
@@ -81,25 +89,21 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteF
     curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
     curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
 
-    if (is_callable($curlWriteFunction)) {
-        curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
-    }
-
     // Max download size management
     curl_setopt($ch, CURLOPT_BUFFERSIZE, 1024*16);
     curl_setopt($ch, CURLOPT_NOPROGRESS, false);
+    if (is_callable($curlHeaderFunction)) {
+        curl_setopt($ch, CURLOPT_HEADERFUNCTION, $curlHeaderFunction);
+    }
+    if (is_callable($curlWriteFunction)) {
+        curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
+    }
     curl_setopt(
         $ch,
         CURLOPT_PROGRESSFUNCTION,
-        function ($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes) {
-            if (version_compare(phpversion(), '5.5', '<')) {
-                // PHP version lower than 5.5
-                // Callback has 4 arguments
-                $downloaded = $arg1;
-            } else {
-                // Callback has 5 arguments
-                $downloaded = $arg2;
-            }
+        function ($arg0, $arg1, $arg2, $arg3, $arg4) use ($maxBytes) {
+            $downloaded = $arg2;
+
             // Non-zero return stops downloading
             return ($downloaded > $maxBytes) ? 1 : 0;
         }
@@ -493,6 +497,46 @@ function is_https($server)
  * Get cURL callback function for CURLOPT_WRITEFUNCTION
  *
  * @param string $charset     to extract from the downloaded page (reference)
+ * @param string $curlGetInfo Optionally overrides curl_getinfo function
+ *
+ * @return Closure
+ */
+function get_curl_header_callback(
+    &$charset,
+    $curlGetInfo = 'curl_getinfo'
+) {
+    $isRedirected = false;
+
+    return function ($ch, $data) use ($curlGetInfo, &$charset, &$isRedirected) {
+        $responseCode = $curlGetInfo($ch, CURLINFO_RESPONSE_CODE);
+        $chunkLength = strlen($data);
+        if (!empty($responseCode) && in_array($responseCode, [301, 302])) {
+            $isRedirected = true;
+            return $chunkLength;
+        }
+        if (!empty($responseCode) && $responseCode !== 200) {
+            return false;
+        }
+        // After a redirection, the content type will keep the previous request value
+        // until it finds the next content-type header.
+        if (! $isRedirected || strpos(strtolower($data), 'content-type') !== false) {
+            $contentType = $curlGetInfo($ch, CURLINFO_CONTENT_TYPE);
+        }
+        if (!empty($contentType) && strpos($contentType, 'text/html') === false) {
+            return false;
+        }
+        if (!empty($contentType) && empty($charset)) {
+            $charset = header_extract_charset($contentType);
+        }
+
+        return $chunkLength;
+    };
+}
+
+/**
+ * Get cURL callback function for CURLOPT_WRITEFUNCTION
+ *
+ * @param string $charset     to extract from the downloaded page (reference)
  * @param string $title       to extract from the downloaded page (reference)
  * @param string $description to extract from the downloaded page (reference)
  * @param string $keywords    to extract from the downloaded page (reference)
@@ -506,10 +550,8 @@ function get_curl_download_callback(
     &$title,
     &$description,
     &$keywords,
-    $retrieveDescription,
-    $curlGetInfo = 'curl_getinfo'
+    $retrieveDescription
 ) {
-    $isRedirected = false;
     $currentChunk = 0;
     $foundChunk = null;
 
@@ -524,37 +566,18 @@ function get_curl_download_callback(
      *
      * @return int|bool length of $data or false if we need to stop the download
      */
-    return function (&$ch, $data) use (
+    return function ($ch, $data) use (
         $retrieveDescription,
-        $curlGetInfo,
         &$charset,
         &$title,
         &$description,
         &$keywords,
-        &$isRedirected,
         &$currentChunk,
         &$foundChunk
     ) {
+        $chunkLength = strlen($data);
         $currentChunk++;
-        $responseCode = $curlGetInfo($ch, CURLINFO_RESPONSE_CODE);
-        if (!empty($responseCode) && in_array($responseCode, [301, 302])) {
-            $isRedirected = true;
-            return strlen($data);
-        }
-        if (!empty($responseCode) && $responseCode !== 200) {
-            return false;
-        }
-        // After a redirection, the content type will keep the previous request value
-        // until it finds the next content-type header.
-        if (! $isRedirected || strpos(strtolower($data), 'content-type') !== false) {
-            $contentType = $curlGetInfo($ch, CURLINFO_CONTENT_TYPE);
-        }
-        if (!empty($contentType) && strpos($contentType, 'text/html') === false) {
-            return false;
-        }
-        if (!empty($contentType) && empty($charset)) {
-            $charset = header_extract_charset($contentType);
-        }
+
         if (empty($charset)) {
             $charset = html_extract_charset($data);
         }
@@ -562,6 +585,10 @@ function get_curl_download_callback(
             $title = html_extract_title($data);
             $foundChunk = ! empty($title) ? $currentChunk : $foundChunk;
         }
+        if (empty($title)) {
+            $title = html_extract_tag('title', $data);
+            $foundChunk = ! empty($title) ? $currentChunk : $foundChunk;
+        }
         if ($retrieveDescription && empty($description)) {
             $description = html_extract_tag('description', $data);
             $foundChunk = ! empty($description) ? $currentChunk : $foundChunk;
@@ -591,6 +618,6 @@ function get_curl_download_callback(
             return false;
         }
 
-        return strlen($data);
+        return $chunkLength;
     };
 }
diff --git a/application/http/MetadataRetriever.php b/application/http/MetadataRetriever.php
new file mode 100644
index 00000000..ba9bd40c
--- /dev/null
+++ b/application/http/MetadataRetriever.php
@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shaarli\Http;
+
+use Shaarli\Config\ConfigManager;
+
+/**
+ * HTTP Tool used to extract metadata from external URL (title, description, etc.).
+ */
+class MetadataRetriever
+{
+    /** @var ConfigManager */
+    protected $conf;
+
+    /** @var HttpAccess */
+    protected $httpAccess;
+
+    public function __construct(ConfigManager $conf, HttpAccess $httpAccess)
+    {
+        $this->conf = $conf;
+        $this->httpAccess = $httpAccess;
+    }
+
+    /**
+     * Retrieve metadata for given URL.
+     *
+     * @return array [
+     *                  'title' => <remote title>,
+     *                  'description' => <remote description>,
+     *                  'tags' => <remote keywords>,
+     *               ]
+     */
+    public function retrieve(string $url): array
+    {
+        $charset = null;
+        $title = null;
+        $description = null;
+        $tags = null;
+        $retrieveDescription = $this->conf->get('general.retrieve_description');
+
+        // Short timeout to keep the application responsive
+        // The callback will fill $charset and $title with data from the downloaded page.
+        $this->httpAccess->getHttpResponse(
+            $url,
+            $this->conf->get('general.download_timeout', 30),
+            $this->conf->get('general.download_max_size', 4194304),
+            $this->httpAccess->getCurlHeaderCallback($charset),
+            $this->httpAccess->getCurlDownloadCallback(
+                $charset,
+                $title,
+                $description,
+                $tags,
+                $retrieveDescription
+            )
+        );
+
+        if (!empty($title) && strtolower($charset) !== 'utf-8') {
+            $title = mb_convert_encoding($title, 'utf-8', $charset);
+        }
+
+        return [
+            'title' => $title,
+            'description' => $description,
+            'tags' => $tags,
+        ];
+    }
+}