Add a setting to retrieve bookmark metadata asynchrounously

  - There is a new standalone script (metadata.js) which requests
    a new controller to get bookmark metadata and fill the form async
  - This feature is enabled with the new setting: general.enable_async_metadata
    (enabled by default)
  - general.retrieve_description is now enabled by default
  - A small rotating loader animation has a been added to bookmark inputs
    when metadata is being retrieved (default template)
  - Custom JS htmlentities has been removed and  mathiasbynens/he
    library is used instead

Fixes #1563

3 files changed, 208 insertions, 35 deletions

diff --git a/tests/container/ContainerBuilderTest.php b/tests/container/ContainerBuilderTest.php
index 5d52daef..3dadc0b9 100644
--- a/tests/container/ContainerBuilderTest.php
+++ b/tests/container/ContainerBuilderTest.php
@@ -12,6 +12,7 @@ use Shaarli\Front\Controller\Visitor\ErrorController;
 use Shaarli\Front\Controller\Visitor\ErrorNotFoundController;
 use Shaarli\History;
 use Shaarli\Http\HttpAccess;
+use Shaarli\Http\MetadataRetriever;
 use Shaarli\Netscape\NetscapeBookmarkUtils;
 use Shaarli\Plugin\PluginManager;
 use Shaarli\Render\PageBuilder;
@@ -72,6 +73,7 @@ class ContainerBuilderTest extends TestCase
         static::assertInstanceOf(History::class, $container->history);
         static::assertInstanceOf(HttpAccess::class, $container->httpAccess);
         static::assertInstanceOf(LoginManager::class, $container->loginManager);
+        static::assertInstanceOf(MetadataRetriever::class, $container->metadataRetriever);
         static::assertInstanceOf(NetscapeBookmarkUtils::class, $container->netscapeBookmarkUtils);
         static::assertInstanceOf(PageBuilder::class, $container->pageBuilder);
         static::assertInstanceOf(PageCacheManager::class, $container->pageCacheManager);
diff --git a/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php b/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php
index 2eb95251..4fd88480 100644
--- a/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php
+++ b/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php
@@ -9,6 +9,7 @@ use Shaarli\Config\ConfigManager;
 use Shaarli\Front\Controller\Admin\FrontAdminControllerMockHelper;
 use Shaarli\Front\Controller\Admin\ManageShaareController;
 use Shaarli\Http\HttpAccess;
+use Shaarli\Http\MetadataRetriever;
 use Shaarli\TestCase;
 use Slim\Http\Request;
 use Slim\Http\Response;
@@ -25,6 +26,7 @@ class DisplayCreateFormTest extends TestCase
         $this->createContainer();
 
         $this->container->httpAccess = $this->createMock(HttpAccess::class);
+        $this->container->metadataRetriever = $this->createMock(MetadataRetriever::class);
         $this->controller = new ManageShaareController($this->container);
     }
 
@@ -32,7 +34,7 @@ class DisplayCreateFormTest extends TestCase
      * Test displaying bookmark create form
      * Ensure that every step of the standard workflow works properly.
      */
-    public function testDisplayCreateFormWithUrl(): void
+    public function testDisplayCreateFormWithUrlAndWithMetadataRetrieval(): void
     {
         $this->container->environment = [
             'HTTP_REFERER' => $referer = 'http://shaarli/subfolder/controller/?searchtag=abc'
@@ -53,40 +55,20 @@ class DisplayCreateFormTest extends TestCase
         });
         $response = new Response();
 
-        $this->container->httpAccess
-            ->expects(static::once())
-            ->method('getCurlDownloadCallback')
-            ->willReturnCallback(
-                function (&$charset, &$title, &$description, &$tags) use (
-                    $remoteTitle,
-                    $remoteDesc,
-                    $remoteTags
-                ): callable {
-                    return function () use (
-                        &$charset,
-                        &$title,
-                        &$description,
-                        &$tags,
-                        $remoteTitle,
-                        $remoteDesc,
-                        $remoteTags
-                    ): void {
-                        $charset = 'ISO-8859-1';
-                        $title = $remoteTitle;
-                        $description = $remoteDesc;
-                        $tags = $remoteTags;
-                    };
-                }
-            )
-        ;
-        $this->container->httpAccess
-            ->expects(static::once())
-            ->method('getHttpResponse')
-            ->with($expectedUrl, 30, 4194304)
-            ->willReturnCallback(function($url, $timeout, $maxBytes, $callback): void {
-                $callback();
-            })
-        ;
+        $this->container->conf = $this->createMock(ConfigManager::class);
+        $this->container->conf->method('get')->willReturnCallback(function (string $param, $default) {
+            if ($param === 'general.enable_async_metadata') {
+                return false;
+            }
+
+            return $default;
+        });
+
+        $this->container->metadataRetriever->expects(static::once())->method('retrieve')->willReturn([
+            'title' => $remoteTitle,
+            'description' => $remoteDesc,
+            'tags' => $remoteTags,
+        ]);
 
         $this->container->bookmarkService
             ->expects(static::once())
@@ -127,6 +109,72 @@ class DisplayCreateFormTest extends TestCase
         static::assertSame($tags, $assignedVariables['tags']);
         static::assertArrayHasKey('source', $assignedVariables);
         static::assertArrayHasKey('default_private_links', $assignedVariables);
+        static::assertArrayHasKey('async_metadata', $assignedVariables);
+        static::assertArrayHasKey('retrieve_description', $assignedVariables);
+    }
+
+    /**
+     * Test displaying bookmark create form without any external metadata retrieval attempt
+     */
+    public function testDisplayCreateFormWithUrlAndWithoutMetadata(): void
+    {
+        $this->container->environment = [
+            'HTTP_REFERER' => $referer = 'http://shaarli/subfolder/controller/?searchtag=abc'
+        ];
+
+        $assignedVariables = [];
+        $this->assignTemplateVars($assignedVariables);
+
+        $url = 'http://url.tld/other?part=3&utm_ad=pay#hash';
+        $expectedUrl = str_replace('&utm_ad=pay', '', $url);
+
+        $request = $this->createMock(Request::class);
+        $request->method('getParam')->willReturnCallback(function (string $key) use ($url): ?string {
+            return $key === 'post' ? $url : null;
+        });
+        $response = new Response();
+
+        $this->container->metadataRetriever->expects(static::never())->method('retrieve');
+
+        $this->container->bookmarkService
+            ->expects(static::once())
+            ->method('bookmarksCountPerTag')
+            ->willReturn($tags = ['tag1' => 2, 'tag2' => 1])
+        ;
+
+        // Make sure that PluginManager hook is triggered
+        $this->container->pluginManager
+            ->expects(static::at(0))
+            ->method('executeHooks')
+            ->willReturnCallback(function (string $hook, array $data): array {
+                static::assertSame('render_editlink', $hook);
+                static::assertSame('', $data['link']['title']);
+                static::assertSame('', $data['link']['description']);
+
+                return $data;
+            })
+        ;
+
+        $result = $this->controller->displayCreateForm($request, $response);
+
+        static::assertSame(200, $result->getStatusCode());
+        static::assertSame('editlink', (string) $result->getBody());
+
+        static::assertSame('Shaare - Shaarli', $assignedVariables['pagetitle']);
+
+        static::assertSame($expectedUrl, $assignedVariables['link']['url']);
+        static::assertSame('', $assignedVariables['link']['title']);
+        static::assertSame('', $assignedVariables['link']['description']);
+        static::assertSame('', $assignedVariables['link']['tags']);
+        static::assertFalse($assignedVariables['link']['private']);
+
+        static::assertTrue($assignedVariables['link_is_new']);
+        static::assertSame($referer, $assignedVariables['http_referer']);
+        static::assertSame($tags, $assignedVariables['tags']);
+        static::assertArrayHasKey('source', $assignedVariables);
+        static::assertArrayHasKey('default_private_links', $assignedVariables);
+        static::assertArrayHasKey('async_metadata', $assignedVariables);
+        static::assertArrayHasKey('retrieve_description', $assignedVariables);
     }
 
     /**
diff --git a/tests/http/MetadataRetrieverTest.php b/tests/http/MetadataRetrieverTest.php
new file mode 100644
index 00000000..2a1838e8
--- /dev/null
+++ b/tests/http/MetadataRetrieverTest.php
@@ -0,0 +1,123 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shaarli\Http;
+
+use PHPUnit\Framework\TestCase;
+use Shaarli\Config\ConfigManager;
+
+class MetadataRetrieverTest extends TestCase
+{
+    /** @var MetadataRetriever */
+    protected $retriever;
+
+    /** @var ConfigManager */
+    protected $conf;
+
+    /** @var HttpAccess */
+    protected $httpAccess;
+
+    public function setUp(): void
+    {
+        $this->conf = $this->createMock(ConfigManager::class);
+        $this->httpAccess = $this->createMock(HttpAccess::class);
+        $this->retriever = new MetadataRetriever($this->conf, $this->httpAccess);
+
+        $this->conf->method('get')->willReturnCallback(function (string $param, $default) {
+            return $default === null ? $param : $default;
+        });
+    }
+
+    /**
+     * Test metadata retrieve() with values returned
+     */
+    public function testFullRetrieval(): void
+    {
+        $url = 'https://domain.tld/link';
+        $remoteTitle = 'Remote Title ';
+        $remoteDesc = 'Sometimes the meta description is relevant.';
+        $remoteTags = 'abc def';
+
+        $expectedResult = [
+            'title' => $remoteTitle,
+            'description' => $remoteDesc,
+            'tags' => $remoteTags,
+        ];
+
+        $this->httpAccess
+            ->expects(static::once())
+            ->method('getCurlDownloadCallback')
+            ->willReturnCallback(
+                function (&$charset, &$title, &$description, &$tags) use (
+                    $remoteTitle,
+                    $remoteDesc,
+                    $remoteTags
+                ): callable {
+                    return function () use (
+                        &$charset,
+                        &$title,
+                        &$description,
+                        &$tags,
+                        $remoteTitle,
+                        $remoteDesc,
+                        $remoteTags
+                    ): void {
+                        $charset = 'ISO-8859-1';
+                        $title = $remoteTitle;
+                        $description = $remoteDesc;
+                        $tags = $remoteTags;
+                    };
+                }
+            )
+        ;
+        $this->httpAccess
+            ->expects(static::once())
+            ->method('getHttpResponse')
+            ->with($url, 30, 4194304)
+            ->willReturnCallback(function($url, $timeout, $maxBytes, $callback): void {
+                $callback();
+            })
+        ;
+
+        $result = $this->retriever->retrieve($url);
+
+        static::assertSame($expectedResult, $result);
+    }
+
+    /**
+     * Test metadata retrieve() without any value
+     */
+    public function testEmptyRetrieval(): void
+    {
+        $url = 'https://domain.tld/link';
+
+        $expectedResult = [
+            'title' => null,
+            'description' => null,
+            'tags' => null,
+        ];
+
+        $this->httpAccess
+            ->expects(static::once())
+            ->method('getCurlDownloadCallback')
+            ->willReturnCallback(
+                function (&$charset, &$title, &$description, &$tags): callable {
+                    return function () use (&$charset, &$title, &$description, &$tags): void {};
+                }
+            )
+        ;
+        $this->httpAccess
+            ->expects(static::once())
+            ->method('getHttpResponse')
+            ->with($url, 30, 4194304)
+            ->willReturnCallback(function($url, $timeout, $maxBytes, $callback): void {
+                $callback();
+            })
+        ;
+
+        $result = $this->retriever->retrieve($url);
+
+        static::assertSame($expectedResult, $result);
+    }
+}