aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorArthurHoaro <arthur@hoa.ro>2020-10-20 10:14:28 +0200
committerGitHub <noreply@github.com>2020-10-20 10:14:28 +0200
commit9b3c1270bcbe4f8e30e0160da8badd43dd94871a (patch)
treef1d87ed084970cf0c3ef99fef8e4ad6dcea423ce
parent552c3b942afe565b780785eab5b2e29c1e800c2e (diff)
parent5334090be04e66da5cb5c3ad487604b3733c5cac (diff)
downloadShaarli-9b3c1270bcbe4f8e30e0160da8badd43dd94871a.tar.gz
Shaarli-9b3c1270bcbe4f8e30e0160da8badd43dd94871a.tar.zst
Shaarli-9b3c1270bcbe4f8e30e0160da8badd43dd94871a.zip
Merge pull request #1567 from ArthurHoaro/feature/async-title-retrieval
-rw-r--r--Makefile1
-rw-r--r--application/config/ConfigManager.php3
-rw-r--r--application/container/ContainerBuilder.php5
-rw-r--r--application/container/ShaarliContainer.php2
-rw-r--r--application/front/controller/admin/ManageShaareController.php36
-rw-r--r--application/front/controller/admin/MetadataController.php29
-rw-r--r--application/http/HttpAccess.php22
-rw-r--r--application/http/HttpUtils.php123
-rw-r--r--application/http/MetadataRetriever.php69
-rw-r--r--assets/common/js/metadata.js39
-rw-r--r--assets/default/js/base.js12
-rw-r--r--assets/default/scss/shaarli.scss51
-rw-r--r--doc/md/Shaarli-configuration.md1
-rw-r--r--index.php2
-rw-r--r--package.json1
-rw-r--r--tests/bookmark/LinkUtilsTest.php223
-rw-r--r--tests/container/ContainerBuilderTest.php2
-rw-r--r--tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php118
-rw-r--r--tests/http/MetadataRetrieverTest.php154
-rw-r--r--tpl/default/editlink.html22
-rw-r--r--webpack.config.js2
-rw-r--r--yarn.lock5
22 files changed, 677 insertions, 245 deletions
diff --git a/Makefile b/Makefile
index 0ff6bd3f..7415887a 100644
--- a/Makefile
+++ b/Makefile
@@ -175,6 +175,7 @@ translate:
175eslint: 175eslint:
176 @yarn run eslint -c .dev/.eslintrc.js assets/vintage/js/ 176 @yarn run eslint -c .dev/.eslintrc.js assets/vintage/js/
177 @yarn run eslint -c .dev/.eslintrc.js assets/default/js/ 177 @yarn run eslint -c .dev/.eslintrc.js assets/default/js/
178 @yarn run eslint -c .dev/.eslintrc.js assets/common/js/
178 179
179### Run CSSLint check against Shaarli's SCSS files 180### Run CSSLint check against Shaarli's SCSS files
180sasslint: 181sasslint:
diff --git a/application/config/ConfigManager.php b/application/config/ConfigManager.php
index 4c98be30..fb085023 100644
--- a/application/config/ConfigManager.php
+++ b/application/config/ConfigManager.php
@@ -366,7 +366,8 @@ class ConfigManager
366 $this->setEmpty('general.links_per_page', 20); 366 $this->setEmpty('general.links_per_page', 20);
367 $this->setEmpty('general.enabled_plugins', self::$DEFAULT_PLUGINS); 367 $this->setEmpty('general.enabled_plugins', self::$DEFAULT_PLUGINS);
368 $this->setEmpty('general.default_note_title', 'Note: '); 368 $this->setEmpty('general.default_note_title', 'Note: ');
369 $this->setEmpty('general.retrieve_description', false); 369 $this->setEmpty('general.retrieve_description', true);
370 $this->setEmpty('general.enable_async_metadata', true);
370 371
371 $this->setEmpty('updates.check_updates', false); 372 $this->setEmpty('updates.check_updates', false);
372 $this->setEmpty('updates.check_updates_branch', 'stable'); 373 $this->setEmpty('updates.check_updates_branch', 'stable');
diff --git a/application/container/ContainerBuilder.php b/application/container/ContainerBuilder.php
index c21d58dd..fd94a1c3 100644
--- a/application/container/ContainerBuilder.php
+++ b/application/container/ContainerBuilder.php
@@ -14,6 +14,7 @@ use Shaarli\Front\Controller\Visitor\ErrorController;
14use Shaarli\Front\Controller\Visitor\ErrorNotFoundController; 14use Shaarli\Front\Controller\Visitor\ErrorNotFoundController;
15use Shaarli\History; 15use Shaarli\History;
16use Shaarli\Http\HttpAccess; 16use Shaarli\Http\HttpAccess;
17use Shaarli\Http\MetadataRetriever;
17use Shaarli\Netscape\NetscapeBookmarkUtils; 18use Shaarli\Netscape\NetscapeBookmarkUtils;
18use Shaarli\Plugin\PluginManager; 19use Shaarli\Plugin\PluginManager;
19use Shaarli\Render\PageBuilder; 20use Shaarli\Render\PageBuilder;
@@ -90,6 +91,10 @@ class ContainerBuilder
90 ); 91 );
91 }; 92 };
92 93
94 $container['metadataRetriever'] = function (ShaarliContainer $container): MetadataRetriever {
95 return new MetadataRetriever($container->conf, $container->httpAccess);
96 };
97
93 $container['pageBuilder'] = function (ShaarliContainer $container): PageBuilder { 98 $container['pageBuilder'] = function (ShaarliContainer $container): PageBuilder {
94 return new PageBuilder( 99 return new PageBuilder(
95 $container->conf, 100 $container->conf,
diff --git a/application/container/ShaarliContainer.php b/application/container/ShaarliContainer.php
index 66e669aa..3a7c238f 100644
--- a/application/container/ShaarliContainer.php
+++ b/application/container/ShaarliContainer.php
@@ -10,6 +10,7 @@ use Shaarli\Feed\FeedBuilder;
10use Shaarli\Formatter\FormatterFactory; 10use Shaarli\Formatter\FormatterFactory;
11use Shaarli\History; 11use Shaarli\History;
12use Shaarli\Http\HttpAccess; 12use Shaarli\Http\HttpAccess;
13use Shaarli\Http\MetadataRetriever;
13use Shaarli\Netscape\NetscapeBookmarkUtils; 14use Shaarli\Netscape\NetscapeBookmarkUtils;
14use Shaarli\Plugin\PluginManager; 15use Shaarli\Plugin\PluginManager;
15use Shaarli\Render\PageBuilder; 16use Shaarli\Render\PageBuilder;
@@ -35,6 +36,7 @@ use Slim\Container;
35 * @property History $history 36 * @property History $history
36 * @property HttpAccess $httpAccess 37 * @property HttpAccess $httpAccess
37 * @property LoginManager $loginManager 38 * @property LoginManager $loginManager
39 * @property MetadataRetriever $metadataRetriever
38 * @property NetscapeBookmarkUtils $netscapeBookmarkUtils 40 * @property NetscapeBookmarkUtils $netscapeBookmarkUtils
39 * @property callable $notFoundHandler Overrides default Slim exception display 41 * @property callable $notFoundHandler Overrides default Slim exception display
40 * @property PageBuilder $pageBuilder 42 * @property PageBuilder $pageBuilder
diff --git a/application/front/controller/admin/ManageShaareController.php b/application/front/controller/admin/ManageShaareController.php
index bb083486..df2f1631 100644
--- a/application/front/controller/admin/ManageShaareController.php
+++ b/application/front/controller/admin/ManageShaareController.php
@@ -53,36 +53,22 @@ class ManageShaareController extends ShaarliAdminController
53 53
54 // If this is an HTTP(S) link, we try go get the page to extract 54 // If this is an HTTP(S) link, we try go get the page to extract
55 // the title (otherwise we will to straight to the edit form.) 55 // the title (otherwise we will to straight to the edit form.)
56 if (empty($title) && strpos(get_url_scheme($url) ?: '', 'http') !== false) { 56 if (true !== $this->container->conf->get('general.enable_async_metadata', true)
57 $retrieveDescription = $this->container->conf->get('general.retrieve_description'); 57 && empty($title)
58 // Short timeout to keep the application responsive 58 && strpos(get_url_scheme($url) ?: '', 'http') !== false
59 // The callback will fill $charset and $title with data from the downloaded page. 59 ) {
60 $this->container->httpAccess->getHttpResponse( 60 $metadata = $this->container->metadataRetriever->retrieve($url);
61 $url,
62 $this->container->conf->get('general.download_timeout', 30),
63 $this->container->conf->get('general.download_max_size', 4194304),
64 $this->container->httpAccess->getCurlDownloadCallback(
65 $charset,
66 $title,
67 $description,
68 $tags,
69 $retrieveDescription
70 )
71 );
72 if (! empty($title) && strtolower($charset) !== 'utf-8' && mb_check_encoding($charset)) {
73 $title = mb_convert_encoding($title, 'utf-8', $charset);
74 }
75 } 61 }
76 62
77 if (empty($url) && empty($title)) { 63 if (empty($url)) {
78 $title = $this->container->conf->get('general.default_note_title', t('Note: ')); 64 $metadata['title'] = $this->container->conf->get('general.default_note_title', t('Note: '));
79 } 65 }
80 66
81 $link = [ 67 $link = [
82 'title' => $title, 68 'title' => $title ?? $metadata['title'] ?? '',
83 'url' => $url ?? '', 69 'url' => $url ?? '',
84 'description' => $description ?? '', 70 'description' => $description ?? $metadata['description'] ?? '',
85 'tags' => $tags ?? '', 71 'tags' => $tags ?? $metadata['tags'] ?? '',
86 'private' => $private, 72 'private' => $private,
87 ]; 73 ];
88 } else { 74 } else {
@@ -352,6 +338,8 @@ class ManageShaareController extends ShaarliAdminController
352 'source' => $request->getParam('source') ?? '', 338 'source' => $request->getParam('source') ?? '',
353 'tags' => $tags, 339 'tags' => $tags,
354 'default_private_links' => $this->container->conf->get('privacy.default_private_links', false), 340 'default_private_links' => $this->container->conf->get('privacy.default_private_links', false),
341 'async_metadata' => $this->container->conf->get('general.enable_async_metadata', true),
342 'retrieve_description' => $this->container->conf->get('general.retrieve_description', false),
355 ]); 343 ]);
356 344
357 $this->executePageHooks('render_editlink', $data, TemplatePage::EDIT_LINK); 345 $this->executePageHooks('render_editlink', $data, TemplatePage::EDIT_LINK);
diff --git a/application/front/controller/admin/MetadataController.php b/application/front/controller/admin/MetadataController.php
new file mode 100644
index 00000000..ff845944
--- /dev/null
+++ b/application/front/controller/admin/MetadataController.php
@@ -0,0 +1,29 @@
1<?php
2
3declare(strict_types=1);
4
5namespace Shaarli\Front\Controller\Admin;
6
7use Slim\Http\Request;
8use Slim\Http\Response;
9
10/**
11 * Controller used to retrieve/update bookmark's metadata.
12 */
13class MetadataController extends ShaarliAdminController
14{
15 /**
16 * GET /admin/metadata/{url} - Attempt to retrieve the bookmark title from provided URL.
17 */
18 public function ajaxRetrieveTitle(Request $request, Response $response): Response
19 {
20 $url = $request->getParam('url');
21
22 // Only try to extract metadata from URL with HTTP(s) scheme
23 if (!empty($url) && strpos(get_url_scheme($url) ?: '', 'http') !== false) {
24 return $response->withJson($this->container->metadataRetriever->retrieve($url));
25 }
26
27 return $response->withJson([]);
28 }
29}
diff --git a/application/http/HttpAccess.php b/application/http/HttpAccess.php
index 81d9e076..646a5264 100644
--- a/application/http/HttpAccess.php
+++ b/application/http/HttpAccess.php
@@ -14,9 +14,14 @@ namespace Shaarli\Http;
14 */ 14 */
15class HttpAccess 15class HttpAccess
16{ 16{
17 public function getHttpResponse($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null) 17 public function getHttpResponse(
18 { 18 $url,
19 return get_http_response($url, $timeout, $maxBytes, $curlWriteFunction); 19 $timeout = 30,
20 $maxBytes = 4194304,
21 $curlHeaderFunction = null,
22 $curlWriteFunction = null
23 ) {
24 return get_http_response($url, $timeout, $maxBytes, $curlHeaderFunction, $curlWriteFunction);
20 } 25 }
21 26
22 public function getCurlDownloadCallback( 27 public function getCurlDownloadCallback(
@@ -24,16 +29,19 @@ class HttpAccess
24 &$title, 29 &$title,
25 &$description, 30 &$description,
26 &$keywords, 31 &$keywords,
27 $retrieveDescription, 32 $retrieveDescription
28 $curlGetInfo = 'curl_getinfo'
29 ) { 33 ) {
30 return get_curl_download_callback( 34 return get_curl_download_callback(
31 $charset, 35 $charset,
32 $title, 36 $title,
33 $description, 37 $description,
34 $keywords, 38 $keywords,
35 $retrieveDescription, 39 $retrieveDescription
36 $curlGetInfo
37 ); 40 );
38 } 41 }
42
43 public function getCurlHeaderCallback(&$charset, $curlGetInfo = 'curl_getinfo')
44 {
45 return get_curl_header_callback($charset, $curlGetInfo);
46 }
39} 47}
diff --git a/application/http/HttpUtils.php b/application/http/HttpUtils.php
index 9f414073..28c12969 100644
--- a/application/http/HttpUtils.php
+++ b/application/http/HttpUtils.php
@@ -6,12 +6,14 @@ use Shaarli\Http\Url;
6 * GET an HTTP URL to retrieve its content 6 * GET an HTTP URL to retrieve its content
7 * Uses the cURL library or a fallback method 7 * Uses the cURL library or a fallback method
8 * 8 *
9 * @param string $url URL to get (http://...) 9 * @param string $url URL to get (http://...)
10 * @param int $timeout network timeout (in seconds) 10 * @param int $timeout network timeout (in seconds)
11 * @param int $maxBytes maximum downloaded bytes (default: 4 MiB) 11 * @param int $maxBytes maximum downloaded bytes (default: 4 MiB)
12 * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION). 12 * @param callable|string $curlHeaderFunction Optional callback called during the download of headers
13 * Can be used to add download conditions on the 13 * (CURLOPT_HEADERFUNCTION)
14 * headers (response code, content type, etc.). 14 * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION).
15 * Can be used to add download conditions on the
16 * headers (response code, content type, etc.).
15 * 17 *
16 * @return array HTTP response headers, downloaded content 18 * @return array HTTP response headers, downloaded content
17 * 19 *
@@ -35,8 +37,13 @@ use Shaarli\Http\Url;
35 * @see http://stackoverflow.com/q/9183178 37 * @see http://stackoverflow.com/q/9183178
36 * @see http://stackoverflow.com/q/1462720 38 * @see http://stackoverflow.com/q/1462720
37 */ 39 */
38function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null) 40function get_http_response(
39{ 41 $url,
42 $timeout = 30,
43 $maxBytes = 4194304,
44 $curlHeaderFunction = null,
45 $curlWriteFunction = null
46) {
40 $urlObj = new Url($url); 47 $urlObj = new Url($url);
41 $cleanUrl = $urlObj->idnToAscii(); 48 $cleanUrl = $urlObj->idnToAscii();
42 49
@@ -70,7 +77,8 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteF
70 // General cURL settings 77 // General cURL settings
71 curl_setopt($ch, CURLOPT_AUTOREFERER, true); 78 curl_setopt($ch, CURLOPT_AUTOREFERER, true);
72 curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); 79 curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
73 curl_setopt($ch, CURLOPT_HEADER, true); 80 // Default header download if the $curlHeaderFunction is not defined
81 curl_setopt($ch, CURLOPT_HEADER, !is_callable($curlHeaderFunction));
74 curl_setopt( 82 curl_setopt(
75 $ch, 83 $ch,
76 CURLOPT_HTTPHEADER, 84 CURLOPT_HTTPHEADER,
@@ -81,25 +89,21 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteF
81 curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); 89 curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
82 curl_setopt($ch, CURLOPT_USERAGENT, $userAgent); 90 curl_setopt($ch, CURLOPT_USERAGENT, $userAgent);
83 91
84 if (is_callable($curlWriteFunction)) {
85 curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
86 }
87
88 // Max download size management 92 // Max download size management
89 curl_setopt($ch, CURLOPT_BUFFERSIZE, 1024*16); 93 curl_setopt($ch, CURLOPT_BUFFERSIZE, 1024*16);
90 curl_setopt($ch, CURLOPT_NOPROGRESS, false); 94 curl_setopt($ch, CURLOPT_NOPROGRESS, false);
95 if (is_callable($curlHeaderFunction)) {
96 curl_setopt($ch, CURLOPT_HEADERFUNCTION, $curlHeaderFunction);
97 }
98 if (is_callable($curlWriteFunction)) {
99 curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction);
100 }
91 curl_setopt( 101 curl_setopt(
92 $ch, 102 $ch,
93 CURLOPT_PROGRESSFUNCTION, 103 CURLOPT_PROGRESSFUNCTION,
94 function ($arg0, $arg1, $arg2, $arg3, $arg4 = 0) use ($maxBytes) { 104 function ($arg0, $arg1, $arg2, $arg3, $arg4) use ($maxBytes) {
95 if (version_compare(phpversion(), '5.5', '<')) { 105 $downloaded = $arg2;
96 // PHP version lower than 5.5 106
97 // Callback has 4 arguments
98 $downloaded = $arg1;
99 } else {
100 // Callback has 5 arguments
101 $downloaded = $arg2;
102 }
103 // Non-zero return stops downloading 107 // Non-zero return stops downloading
104 return ($downloaded > $maxBytes) ? 1 : 0; 108 return ($downloaded > $maxBytes) ? 1 : 0;
105 } 109 }
@@ -493,6 +497,46 @@ function is_https($server)
493 * Get cURL callback function for CURLOPT_WRITEFUNCTION 497 * Get cURL callback function for CURLOPT_WRITEFUNCTION
494 * 498 *
495 * @param string $charset to extract from the downloaded page (reference) 499 * @param string $charset to extract from the downloaded page (reference)
500 * @param string $curlGetInfo Optionally overrides curl_getinfo function
501 *
502 * @return Closure
503 */
504function get_curl_header_callback(
505 &$charset,
506 $curlGetInfo = 'curl_getinfo'
507) {
508 $isRedirected = false;
509
510 return function ($ch, $data) use ($curlGetInfo, &$charset, &$isRedirected) {
511 $responseCode = $curlGetInfo($ch, CURLINFO_RESPONSE_CODE);
512 $chunkLength = strlen($data);
513 if (!empty($responseCode) && in_array($responseCode, [301, 302])) {
514 $isRedirected = true;
515 return $chunkLength;
516 }
517 if (!empty($responseCode) && $responseCode !== 200) {
518 return false;
519 }
520 // After a redirection, the content type will keep the previous request value
521 // until it finds the next content-type header.
522 if (! $isRedirected || strpos(strtolower($data), 'content-type') !== false) {
523 $contentType = $curlGetInfo($ch, CURLINFO_CONTENT_TYPE);
524 }
525 if (!empty($contentType) && strpos($contentType, 'text/html') === false) {
526 return false;
527 }
528 if (!empty($contentType) && empty($charset)) {
529 $charset = header_extract_charset($contentType);
530 }
531
532 return $chunkLength;
533 };
534}
535
536/**
537 * Get cURL callback function for CURLOPT_WRITEFUNCTION
538 *
539 * @param string $charset to extract from the downloaded page (reference)
496 * @param string $title to extract from the downloaded page (reference) 540 * @param string $title to extract from the downloaded page (reference)
497 * @param string $description to extract from the downloaded page (reference) 541 * @param string $description to extract from the downloaded page (reference)
498 * @param string $keywords to extract from the downloaded page (reference) 542 * @param string $keywords to extract from the downloaded page (reference)
@@ -506,10 +550,8 @@ function get_curl_download_callback(
506 &$title, 550 &$title,
507 &$description, 551 &$description,
508 &$keywords, 552 &$keywords,
509 $retrieveDescription, 553 $retrieveDescription
510 $curlGetInfo = 'curl_getinfo'
511) { 554) {
512 $isRedirected = false;
513 $currentChunk = 0; 555 $currentChunk = 0;
514 $foundChunk = null; 556 $foundChunk = null;
515 557
@@ -524,37 +566,18 @@ function get_curl_download_callback(
524 * 566 *
525 * @return int|bool length of $data or false if we need to stop the download 567 * @return int|bool length of $data or false if we need to stop the download
526 */ 568 */
527 return function (&$ch, $data) use ( 569 return function ($ch, $data) use (
528 $retrieveDescription, 570 $retrieveDescription,
529 $curlGetInfo,
530 &$charset, 571 &$charset,
531 &$title, 572 &$title,
532 &$description, 573 &$description,
533 &$keywords, 574 &$keywords,
534 &$isRedirected,
535 &$currentChunk, 575 &$currentChunk,
536 &$foundChunk 576 &$foundChunk
537 ) { 577 ) {
578 $chunkLength = strlen($data);
538 $currentChunk++; 579 $currentChunk++;
539 $responseCode = $curlGetInfo($ch, CURLINFO_RESPONSE_CODE); 580
540 if (!empty($responseCode) && in_array($responseCode, [301, 302])) {
541 $isRedirected = true;
542 return strlen($data);
543 }
544 if (!empty($responseCode) && $responseCode !== 200) {
545 return false;
546 }
547 // After a redirection, the content type will keep the previous request value
548 // until it finds the next content-type header.
549 if (! $isRedirected || strpos(strtolower($data), 'content-type') !== false) {
550 $contentType = $curlGetInfo($ch, CURLINFO_CONTENT_TYPE);
551 }
552 if (!empty($contentType) && strpos($contentType, 'text/html') === false) {
553 return false;
554 }
555 if (!empty($contentType) && empty($charset)) {
556 $charset = header_extract_charset($contentType);
557 }
558 if (empty($charset)) { 581 if (empty($charset)) {
559 $charset = html_extract_charset($data); 582 $charset = html_extract_charset($data);
560 } 583 }
@@ -562,6 +585,10 @@ function get_curl_download_callback(
562 $title = html_extract_title($data); 585 $title = html_extract_title($data);
563 $foundChunk = ! empty($title) ? $currentChunk : $foundChunk; 586 $foundChunk = ! empty($title) ? $currentChunk : $foundChunk;
564 } 587 }
588 if (empty($title)) {
589 $title = html_extract_tag('title', $data);
590 $foundChunk = ! empty($title) ? $currentChunk : $foundChunk;
591 }
565 if ($retrieveDescription && empty($description)) { 592 if ($retrieveDescription && empty($description)) {
566 $description = html_extract_tag('description', $data); 593 $description = html_extract_tag('description', $data);
567 $foundChunk = ! empty($description) ? $currentChunk : $foundChunk; 594 $foundChunk = ! empty($description) ? $currentChunk : $foundChunk;
@@ -591,6 +618,6 @@ function get_curl_download_callback(
591 return false; 618 return false;
592 } 619 }
593 620
594 return strlen($data); 621 return $chunkLength;
595 }; 622 };
596} 623}
diff --git a/application/http/MetadataRetriever.php b/application/http/MetadataRetriever.php
new file mode 100644
index 00000000..ba9bd40c
--- /dev/null
+++ b/application/http/MetadataRetriever.php
@@ -0,0 +1,69 @@
1<?php
2
3declare(strict_types=1);
4
5namespace Shaarli\Http;
6
7use Shaarli\Config\ConfigManager;
8
9/**
10 * HTTP Tool used to extract metadata from external URL (title, description, etc.).
11 */
12class MetadataRetriever
13{
14 /** @var ConfigManager */
15 protected $conf;
16
17 /** @var HttpAccess */
18 protected $httpAccess;
19
20 public function __construct(ConfigManager $conf, HttpAccess $httpAccess)
21 {
22 $this->conf = $conf;
23 $this->httpAccess = $httpAccess;
24 }
25
26 /**
27 * Retrieve metadata for given URL.
28 *
29 * @return array [
30 * 'title' => <remote title>,
31 * 'description' => <remote description>,
32 * 'tags' => <remote keywords>,
33 * ]
34 */
35 public function retrieve(string $url): array
36 {
37 $charset = null;
38 $title = null;
39 $description = null;
40 $tags = null;
41 $retrieveDescription = $this->conf->get('general.retrieve_description');
42
43 // Short timeout to keep the application responsive
44 // The callback will fill $charset and $title with data from the downloaded page.
45 $this->httpAccess->getHttpResponse(
46 $url,
47 $this->conf->get('general.download_timeout', 30),
48 $this->conf->get('general.download_max_size', 4194304),
49 $this->httpAccess->getCurlHeaderCallback($charset),
50 $this->httpAccess->getCurlDownloadCallback(
51 $charset,
52 $title,
53 $description,
54 $tags,
55 $retrieveDescription
56 )
57 );
58
59 if (!empty($title) && strtolower($charset) !== 'utf-8') {
60 $title = mb_convert_encoding($title, 'utf-8', $charset);
61 }
62
63 return [
64 'title' => $title,
65 'description' => $description,
66 'tags' => $tags,
67 ];
68 }
69}
diff --git a/assets/common/js/metadata.js b/assets/common/js/metadata.js
new file mode 100644
index 00000000..5200b481
--- /dev/null
+++ b/assets/common/js/metadata.js
@@ -0,0 +1,39 @@
1import he from 'he';
2
3function clearLoaders(loaders) {
4 if (loaders != null && loaders.length > 0) {
5 [...loaders].forEach((loader) => {
6 loader.classList.remove('loading-input');
7 });
8 }
9}
10
11(() => {
12 const loaders = document.querySelectorAll('.loading-input');
13 const inputTitle = document.querySelector('input[name="lf_title"]');
14 if (inputTitle != null && inputTitle.value.length > 0) {
15 clearLoaders(loaders);
16 return;
17 }
18
19 const url = document.querySelector('input[name="lf_url"]').value;
20 const basePath = document.querySelector('input[name="js_base_path"]').value;
21
22 const xhr = new XMLHttpRequest();
23 xhr.open('GET', `${basePath}/admin/metadata?url=${encodeURI(url)}`, true);
24 xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
25 xhr.onload = () => {
26 const result = JSON.parse(xhr.response);
27 Object.keys(result).forEach((key) => {
28 if (result[key] !== null && result[key].length) {
29 const element = document.querySelector(`input[name="lf_${key}"], textarea[name="lf_${key}"]`);
30 if (element != null && element.value.length === 0) {
31 element.value = he.decode(result[key]);
32 }
33 }
34 });
35 clearLoaders(loaders);
36 };
37
38 xhr.send();
39})();
diff --git a/assets/default/js/base.js b/assets/default/js/base.js
index aadffc13..7f6b9637 100644
--- a/assets/default/js/base.js
+++ b/assets/default/js/base.js
@@ -1,4 +1,5 @@
1import Awesomplete from 'awesomplete'; 1import Awesomplete from 'awesomplete';
2import he from 'he';
2 3
3/** 4/**
4 * Find a parent element according to its tag and its attributes 5 * Find a parent element according to its tag and its attributes
@@ -96,15 +97,6 @@ function updateAwesompleteList(selector, tags, instances) {
96} 97}
97 98
98/** 99/**
99 * html_entities in JS
100 *
101 * @see http://stackoverflow.com/questions/18749591/encode-html-entities-in-javascript
102 */
103function htmlEntities(str) {
104 return str.replace(/[\u00A0-\u9999<>&]/gim, (i) => `&#${i.charCodeAt(0)};`);
105}
106
107/**
108 * Add the class 'hidden' to city options not attached to the current selected continent. 100 * Add the class 'hidden' to city options not attached to the current selected continent.
109 * 101 *
110 * @param cities List of <option> elements 102 * @param cities List of <option> elements
@@ -569,7 +561,7 @@ function init(description) {
569 input.setAttribute('name', totag); 561 input.setAttribute('name', totag);
570 input.setAttribute('value', totag); 562 input.setAttribute('value', totag);
571 findParent(input, 'div', { class: 'rename-tag-form' }).style.display = 'none'; 563 findParent(input, 'div', { class: 'rename-tag-form' }).style.display = 'none';
572 block.querySelector('a.tag-link').innerHTML = htmlEntities(totag); 564 block.querySelector('a.tag-link').innerHTML = he.encode(totag);
573 block 565 block
574 .querySelector('a.tag-link') 566 .querySelector('a.tag-link')
575 .setAttribute('href', `${basePath}/?searchtags=${encodeURIComponent(totag)}`); 567 .setAttribute('href', `${basePath}/?searchtags=${encodeURIComponent(totag)}`);
diff --git a/assets/default/scss/shaarli.scss b/assets/default/scss/shaarli.scss
index 2f49bbd2..286ac83b 100644
--- a/assets/default/scss/shaarli.scss
+++ b/assets/default/scss/shaarli.scss
@@ -1273,6 +1273,57 @@ form {
1273 } 1273 }
1274} 1274}
1275 1275
1276.loading-input {
1277 position: relative;
1278
1279 @keyframes around {
1280 0% {
1281 transform: rotate(0deg);
1282 }
1283
1284 100% {
1285 transform: rotate(360deg);
1286 }
1287 }
1288
1289 .icon-container {
1290 position: absolute;
1291 right: 60px;
1292 top: calc(50% - 10px);
1293 }
1294
1295 .loader {
1296 position: relative;
1297 height: 20px;
1298 width: 20px;
1299 display: inline-block;
1300 animation: around 5.4s infinite;
1301
1302 &::after,
1303 &::before {
1304 content: "";
1305 background: $form-input-background;
1306 position: absolute;
1307 display: inline-block;
1308 width: 100%;
1309 height: 100%;
1310 border-width: 2px;
1311 border-color: #333 #333 transparent transparent;
1312 border-style: solid;
1313 border-radius: 20px;
1314 box-sizing: border-box;
1315 top: 0;
1316 left: 0;
1317 animation: around 0.7s ease-in-out infinite;
1318 }
1319
1320 &::after {
1321 animation: around 0.7s ease-in-out 0.1s infinite;
1322 background: transparent;
1323 }
1324 }
1325}
1326
1276// LOGIN 1327// LOGIN
1277.login-form-container { 1328.login-form-container {
1278 .remember-me { 1329 .remember-me {
diff --git a/doc/md/Shaarli-configuration.md b/doc/md/Shaarli-configuration.md
index 263fb761..dbfc3da9 100644
--- a/doc/md/Shaarli-configuration.md
+++ b/doc/md/Shaarli-configuration.md
@@ -150,6 +150,7 @@ _These settings should not be edited_
150- **timezone**: See [the list of supported timezones](http://php.net/manual/en/timezones.php). 150- **timezone**: See [the list of supported timezones](http://php.net/manual/en/timezones.php).
151- **enabled_plugins**: List of enabled plugins. 151- **enabled_plugins**: List of enabled plugins.
152- **default_note_title**: Default title of a new note. 152- **default_note_title**: Default title of a new note.
153- **enable_async_metadata** (boolean): Retrieve external bookmark metadata asynchronously to prevent bookmark creation slowdown.
153- **retrieve_description** (boolean): If set to true, for every new Shaare Shaarli will try to retrieve the description and keywords from the HTML meta tags. 154- **retrieve_description** (boolean): If set to true, for every new Shaare Shaarli will try to retrieve the description and keywords from the HTML meta tags.
154- **root_url**: Overrides automatic discovery of Shaarli instance's URL (e.g.) `https://sub.domain.tld/shaarli-folder/`. 155- **root_url**: Overrides automatic discovery of Shaarli instance's URL (e.g.) `https://sub.domain.tld/shaarli-folder/`.
155 156
diff --git a/index.php b/index.php
index b10397dd..220847f5 100644
--- a/index.php
+++ b/index.php
@@ -129,7 +129,7 @@ $app->group('/admin', function () {
129 $this->post('/plugins', '\Shaarli\Front\Controller\Admin\PluginsController:save'); 129 $this->post('/plugins', '\Shaarli\Front\Controller\Admin\PluginsController:save');
130 $this->get('/token', '\Shaarli\Front\Controller\Admin\TokenController:getToken'); 130 $this->get('/token', '\Shaarli\Front\Controller\Admin\TokenController:getToken');
131 $this->get('/thumbnails', '\Shaarli\Front\Controller\Admin\ThumbnailsController:index'); 131 $this->get('/thumbnails', '\Shaarli\Front\Controller\Admin\ThumbnailsController:index');
132 132 $this->get('/metadata', '\Shaarli\Front\Controller\Admin\MetadataController:ajaxRetrieveTitle');
133 $this->get('/visibility/{visibility}', '\Shaarli\Front\Controller\Admin\SessionFilterController:visibility'); 133 $this->get('/visibility/{visibility}', '\Shaarli\Front\Controller\Admin\SessionFilterController:visibility');
134})->add('\Shaarli\Front\ShaarliAdminMiddleware'); 134})->add('\Shaarli\Front\ShaarliAdminMiddleware');
135 135
diff --git a/package.json b/package.json
index 8a24512a..b879b223 100644
--- a/package.json
+++ b/package.json
@@ -7,6 +7,7 @@
7 "awesomplete": "^1.1.2", 7 "awesomplete": "^1.1.2",
8 "blazy": "^1.8.2", 8 "blazy": "^1.8.2",
9 "fork-awesome": "^1.1.7", 9 "fork-awesome": "^1.1.7",
10 "he": "^1.2.0",
10 "pure-extras": "^1.0.0", 11 "pure-extras": "^1.0.0",
11 "purecss": "^1.0.0" 12 "purecss": "^1.0.0"
12 }, 13 },
diff --git a/tests/bookmark/LinkUtilsTest.php b/tests/bookmark/LinkUtilsTest.php
index 29941c8c..3321242f 100644
--- a/tests/bookmark/LinkUtilsTest.php
+++ b/tests/bookmark/LinkUtilsTest.php
@@ -216,60 +216,91 @@ class LinkUtilsTest extends TestCase
216 } 216 }
217 217
218 /** 218 /**
219 * Test the header callback with valid value
220 */
221 public function testCurlHeaderCallbackOk(): void
222 {
223 $callback = get_curl_header_callback($charset, 'ut_curl_getinfo_ok');
224 $data = [
225 'HTTP/1.1 200 OK',
226 'Server: GitHub.com',
227 'Date: Sat, 28 Oct 2017 12:01:33 GMT',
228 'Content-Type: text/html; charset=utf-8',
229 'Status: 200 OK',
230 ];
231
232 foreach ($data as $chunk) {
233 static::assertIsInt($callback(null, $chunk));
234 }
235
236 static::assertSame('utf-8', $charset);
237 }
238
239 /**
219 * Test the download callback with valid value 240 * Test the download callback with valid value
220 */ 241 */
221 public function testCurlDownloadCallbackOk() 242 public function testCurlDownloadCallbackOk(): void
222 { 243 {
244 $charset = 'utf-8';
223 $callback = get_curl_download_callback( 245 $callback = get_curl_download_callback(
224 $charset, 246 $charset,
225 $title, 247 $title,
226 $desc, 248 $desc,
227 $keywords, 249 $keywords,
228 false, 250 false
229 'ut_curl_getinfo_ok'
230 ); 251 );
252
231 $data = [ 253 $data = [
232 'HTTP/1.1 200 OK', 254 'th=device-width">'
233 'Server: GitHub.com',
234 'Date: Sat, 28 Oct 2017 12:01:33 GMT',
235 'Content-Type: text/html; charset=utf-8',
236 'Status: 200 OK',
237 'end' => 'th=device-width">'
238 . '<title>Refactoring · GitHub</title>' 255 . '<title>Refactoring · GitHub</title>'
239 . '<link rel="search" type="application/opensea', 256 . '<link rel="search" type="application/opensea',
240 '<title>ignored</title>' 257 '<title>ignored</title>'
241 . '<meta name="description" content="desc" />' 258 . '<meta name="description" content="desc" />'
242 . '<meta name="keywords" content="key1,key2" />', 259 . '<meta name="keywords" content="key1,key2" />',
243 ]; 260 ];
244 foreach ($data as $key => $line) { 261
245 $ignore = null; 262 foreach ($data as $chunk) {
246 $expected = $key !== 'end' ? strlen($line) : false; 263 static::assertSame(strlen($chunk), $callback(null, $chunk));
247 $this->assertEquals($expected, $callback($ignore, $line));
248 if ($expected === false) {
249 break;
250 }
251 } 264 }
252 $this->assertEquals('utf-8', $charset); 265
253 $this->assertEquals('Refactoring · GitHub', $title); 266 static::assertSame('utf-8', $charset);
254 $this->assertEmpty($desc); 267 static::assertSame('Refactoring · GitHub', $title);
255 $this->assertEmpty($keywords); 268 static::assertEmpty($desc);
269 static::assertEmpty($keywords);
270 }
271
272 /**
273 * Test the header callback with valid value
274 */
275 public function testCurlHeaderCallbackNoCharset(): void
276 {
277 $callback = get_curl_header_callback($charset, 'ut_curl_getinfo_no_charset');
278 $data = [
279 'HTTP/1.1 200 OK',
280 ];
281
282 foreach ($data as $chunk) {
283 static::assertSame(strlen($chunk), $callback(null, $chunk));
284 }
285
286 static::assertFalse($charset);
256 } 287 }
257 288
258 /** 289 /**
259 * Test the download callback with valid values and no charset 290 * Test the download callback with valid values and no charset
260 */ 291 */
261 public function testCurlDownloadCallbackOkNoCharset() 292 public function testCurlDownloadCallbackOkNoCharset(): void
262 { 293 {
294 $charset = null;
263 $callback = get_curl_download_callback( 295 $callback = get_curl_download_callback(
264 $charset, 296 $charset,
265 $title, 297 $title,
266 $desc, 298 $desc,
267 $keywords, 299 $keywords,
268 false, 300 false
269 'ut_curl_getinfo_no_charset'
270 ); 301 );
302
271 $data = [ 303 $data = [
272 'HTTP/1.1 200 OK',
273 'end' => 'th=device-width">' 304 'end' => 'th=device-width">'
274 . '<title>Refactoring · GitHub</title>' 305 . '<title>Refactoring · GitHub</title>'
275 . '<link rel="search" type="application/opensea', 306 . '<link rel="search" type="application/opensea',
@@ -277,10 +308,11 @@ class LinkUtilsTest extends TestCase
277 . '<meta name="description" content="desc" />' 308 . '<meta name="description" content="desc" />'
278 . '<meta name="keywords" content="key1,key2" />', 309 . '<meta name="keywords" content="key1,key2" />',
279 ]; 310 ];
280 foreach ($data as $key => $line) { 311
281 $ignore = null; 312 foreach ($data as $chunk) {
282 $this->assertEquals(strlen($line), $callback($ignore, $line)); 313 static::assertSame(strlen($chunk), $callback(null, $chunk));
283 } 314 }
315
284 $this->assertEmpty($charset); 316 $this->assertEmpty($charset);
285 $this->assertEquals('Refactoring · GitHub', $title); 317 $this->assertEquals('Refactoring · GitHub', $title);
286 $this->assertEmpty($desc); 318 $this->assertEmpty($desc);
@@ -290,18 +322,18 @@ class LinkUtilsTest extends TestCase
290 /** 322 /**
291 * Test the download callback with valid values and no charset 323 * Test the download callback with valid values and no charset
292 */ 324 */
293 public function testCurlDownloadCallbackOkHtmlCharset() 325 public function testCurlDownloadCallbackOkHtmlCharset(): void
294 { 326 {
327 $charset = null;
295 $callback = get_curl_download_callback( 328 $callback = get_curl_download_callback(
296 $charset, 329 $charset,
297 $title, 330 $title,
298 $desc, 331 $desc,
299 $keywords, 332 $keywords,
300 false, 333 false
301 'ut_curl_getinfo_no_charset'
302 ); 334 );
335
303 $data = [ 336 $data = [
304 'HTTP/1.1 200 OK',
305 '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />', 337 '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />',
306 'end' => 'th=device-width">' 338 'end' => 'th=device-width">'
307 . '<title>Refactoring · GitHub</title>' 339 . '<title>Refactoring · GitHub</title>'
@@ -310,14 +342,10 @@ class LinkUtilsTest extends TestCase
310 . '<meta name="description" content="desc" />' 342 . '<meta name="description" content="desc" />'
311 . '<meta name="keywords" content="key1,key2" />', 343 . '<meta name="keywords" content="key1,key2" />',
312 ]; 344 ];
313 foreach ($data as $key => $line) { 345 foreach ($data as $chunk) {
314 $ignore = null; 346 static::assertSame(strlen($chunk), $callback(null, $chunk));
315 $expected = $key !== 'end' ? strlen($line) : false;
316 $this->assertEquals($expected, $callback($ignore, $line));
317 if ($expected === false) {
318 break;
319 }
320 } 347 }
348
321 $this->assertEquals('utf-8', $charset); 349 $this->assertEquals('utf-8', $charset);
322 $this->assertEquals('Refactoring · GitHub', $title); 350 $this->assertEquals('Refactoring · GitHub', $title);
323 $this->assertEmpty($desc); 351 $this->assertEmpty($desc);
@@ -327,25 +355,26 @@ class LinkUtilsTest extends TestCase
327 /** 355 /**
328 * Test the download callback with valid values and no title 356 * Test the download callback with valid values and no title
329 */ 357 */
330 public function testCurlDownloadCallbackOkNoTitle() 358 public function testCurlDownloadCallbackOkNoTitle(): void
331 { 359 {
360 $charset = 'utf-8';
332 $callback = get_curl_download_callback( 361 $callback = get_curl_download_callback(
333 $charset, 362 $charset,
334 $title, 363 $title,
335 $desc, 364 $desc,
336 $keywords, 365 $keywords,
337 false, 366 false
338 'ut_curl_getinfo_ok'
339 ); 367 );
368
340 $data = [ 369 $data = [
341 'HTTP/1.1 200 OK',
342 'end' => 'th=device-width">Refactoring · GitHub<link rel="search" type="application/opensea', 370 'end' => 'th=device-width">Refactoring · GitHub<link rel="search" type="application/opensea',
343 'ignored', 371 'ignored',
344 ]; 372 ];
345 foreach ($data as $key => $line) { 373
346 $ignore = null; 374 foreach ($data as $chunk) {
347 $this->assertEquals(strlen($line), $callback($ignore, $line)); 375 static::assertSame(strlen($chunk), $callback(null, $chunk));
348 } 376 }
377
349 $this->assertEquals('utf-8', $charset); 378 $this->assertEquals('utf-8', $charset);
350 $this->assertEmpty($title); 379 $this->assertEmpty($title);
351 $this->assertEmpty($desc); 380 $this->assertEmpty($desc);
@@ -353,81 +382,55 @@ class LinkUtilsTest extends TestCase
353 } 382 }
354 383
355 /** 384 /**
356 * Test the download callback with an invalid content type. 385 * Test the header callback with an invalid content type.
357 */ 386 */
358 public function testCurlDownloadCallbackInvalidContentType() 387 public function testCurlHeaderCallbackInvalidContentType(): void
359 { 388 {
360 $callback = get_curl_download_callback( 389 $callback = get_curl_header_callback($charset, 'ut_curl_getinfo_ct_ko');
361 $charset, 390 $data = [
362 $title, 391 'HTTP/1.1 200 OK',
363 $desc, 392 ];
364 $keywords, 393
365 false, 394 static::assertFalse($callback(null, $data[0]));
366 'ut_curl_getinfo_ct_ko' 395 static::assertNull($charset);
367 );
368 $ignore = null;
369 $this->assertFalse($callback($ignore, ''));
370 $this->assertEmpty($charset);
371 $this->assertEmpty($title);
372 } 396 }
373 397
374 /** 398 /**
375 * Test the download callback with an invalid response code. 399 * Test the header callback with an invalid response code.
376 */ 400 */
377 public function testCurlDownloadCallbackInvalidResponseCode() 401 public function testCurlHeaderCallbackInvalidResponseCode(): void
378 { 402 {
379 $callback = $callback = get_curl_download_callback( 403 $callback = get_curl_header_callback($charset, 'ut_curl_getinfo_rc_ko');
380 $charset, 404
381 $title, 405 static::assertFalse($callback(null, ''));
382 $desc, 406 static::assertNull($charset);
383 $keywords,
384 false,
385 'ut_curl_getinfo_rc_ko'
386 );
387 $ignore = null;
388 $this->assertFalse($callback($ignore, ''));
389 $this->assertEmpty($charset);
390 $this->assertEmpty($title);
391 } 407 }
392 408
393 /** 409 /**
394 * Test the download callback with an invalid content type and response code. 410 * Test the header callback with an invalid content type and response code.
395 */ 411 */
396 public function testCurlDownloadCallbackInvalidContentTypeAndResponseCode() 412 public function testCurlHeaderCallbackInvalidContentTypeAndResponseCode(): void
397 { 413 {
398 $callback = $callback = get_curl_download_callback( 414 $callback = get_curl_header_callback($charset, 'ut_curl_getinfo_rs_ct_ko');
399 $charset, 415
400 $title, 416 static::assertFalse($callback(null, ''));
401 $desc, 417 static::assertNull($charset);
402 $keywords,
403 false,
404 'ut_curl_getinfo_rs_ct_ko'
405 );
406 $ignore = null;
407 $this->assertFalse($callback($ignore, ''));
408 $this->assertEmpty($charset);
409 $this->assertEmpty($title);
410 } 418 }
411 419
412 /** 420 /**
413 * Test the download callback with valid value, and retrieve_description option enabled. 421 * Test the download callback with valid value, and retrieve_description option enabled.
414 */ 422 */
415 public function testCurlDownloadCallbackOkWithDesc() 423 public function testCurlDownloadCallbackOkWithDesc(): void
416 { 424 {
425 $charset = 'utf-8';
417 $callback = get_curl_download_callback( 426 $callback = get_curl_download_callback(
418 $charset, 427 $charset,
419 $title, 428 $title,
420 $desc, 429 $desc,
421 $keywords, 430 $keywords,
422 true, 431 true
423 'ut_curl_getinfo_ok'
424 ); 432 );
425 $data = [ 433 $data = [
426 'HTTP/1.1 200 OK',
427 'Server: GitHub.com',
428 'Date: Sat, 28 Oct 2017 12:01:33 GMT',
429 'Content-Type: text/html; charset=utf-8',
430 'Status: 200 OK',
431 'th=device-width">' 434 'th=device-width">'
432 . '<title>Refactoring · GitHub</title>' 435 . '<title>Refactoring · GitHub</title>'
433 . '<link rel="search" type="application/opensea', 436 . '<link rel="search" type="application/opensea',
@@ -435,14 +438,11 @@ class LinkUtilsTest extends TestCase
435 . '<meta name="description" content="link desc" />' 438 . '<meta name="description" content="link desc" />'
436 . '<meta name="keywords" content="key1,key2" />', 439 . '<meta name="keywords" content="key1,key2" />',
437 ]; 440 ];
438 foreach ($data as $key => $line) { 441
439 $ignore = null; 442 foreach ($data as $chunk) {
440 $expected = $key !== 'end' ? strlen($line) : false; 443 static::assertSame(strlen($chunk), $callback(null, $chunk));
441 $this->assertEquals($expected, $callback($ignore, $line));
442 if ($expected === false) {
443 break;
444 }
445 } 444 }
445
446 $this->assertEquals('utf-8', $charset); 446 $this->assertEquals('utf-8', $charset);
447 $this->assertEquals('Refactoring · GitHub', $title); 447 $this->assertEquals('Refactoring · GitHub', $title);
448 $this->assertEquals('link desc', $desc); 448 $this->assertEquals('link desc', $desc);
@@ -453,8 +453,9 @@ class LinkUtilsTest extends TestCase
453 * Test the download callback with valid value, and retrieve_description option enabled, 453 * Test the download callback with valid value, and retrieve_description option enabled,
454 * but no desc or keyword defined in the page. 454 * but no desc or keyword defined in the page.
455 */ 455 */
456 public function testCurlDownloadCallbackOkWithDescNotFound() 456 public function testCurlDownloadCallbackOkWithDescNotFound(): void
457 { 457 {
458 $charset = 'utf-8';
458 $callback = get_curl_download_callback( 459 $callback = get_curl_download_callback(
459 $charset, 460 $charset,
460 $title, 461 $title,
@@ -464,24 +465,16 @@ class LinkUtilsTest extends TestCase
464 'ut_curl_getinfo_ok' 465 'ut_curl_getinfo_ok'
465 ); 466 );
466 $data = [ 467 $data = [
467 'HTTP/1.1 200 OK',
468 'Server: GitHub.com',
469 'Date: Sat, 28 Oct 2017 12:01:33 GMT',
470 'Content-Type: text/html; charset=utf-8',
471 'Status: 200 OK',
472 'th=device-width">' 468 'th=device-width">'
473 . '<title>Refactoring · GitHub</title>' 469 . '<title>Refactoring · GitHub</title>'
474 . '<link rel="search" type="application/opensea', 470 . '<link rel="search" type="application/opensea',
475 'end' => '<title>ignored</title>', 471 'end' => '<title>ignored</title>',
476 ]; 472 ];
477 foreach ($data as $key => $line) { 473
478 $ignore = null; 474 foreach ($data as $chunk) {
479 $expected = $key !== 'end' ? strlen($line) : false; 475 static::assertSame(strlen($chunk), $callback(null, $chunk));
480 $this->assertEquals($expected, $callback($ignore, $line));
481 if ($expected === false) {
482 break;
483 }
484 } 476 }
477
485 $this->assertEquals('utf-8', $charset); 478 $this->assertEquals('utf-8', $charset);
486 $this->assertEquals('Refactoring · GitHub', $title); 479 $this->assertEquals('Refactoring · GitHub', $title);
487 $this->assertEmpty($desc); 480 $this->assertEmpty($desc);
diff --git a/tests/container/ContainerBuilderTest.php b/tests/container/ContainerBuilderTest.php
index 5d52daef..3dadc0b9 100644
--- a/tests/container/ContainerBuilderTest.php
+++ b/tests/container/ContainerBuilderTest.php
@@ -12,6 +12,7 @@ use Shaarli\Front\Controller\Visitor\ErrorController;
12use Shaarli\Front\Controller\Visitor\ErrorNotFoundController; 12use Shaarli\Front\Controller\Visitor\ErrorNotFoundController;
13use Shaarli\History; 13use Shaarli\History;
14use Shaarli\Http\HttpAccess; 14use Shaarli\Http\HttpAccess;
15use Shaarli\Http\MetadataRetriever;
15use Shaarli\Netscape\NetscapeBookmarkUtils; 16use Shaarli\Netscape\NetscapeBookmarkUtils;
16use Shaarli\Plugin\PluginManager; 17use Shaarli\Plugin\PluginManager;
17use Shaarli\Render\PageBuilder; 18use Shaarli\Render\PageBuilder;
@@ -72,6 +73,7 @@ class ContainerBuilderTest extends TestCase
72 static::assertInstanceOf(History::class, $container->history); 73 static::assertInstanceOf(History::class, $container->history);
73 static::assertInstanceOf(HttpAccess::class, $container->httpAccess); 74 static::assertInstanceOf(HttpAccess::class, $container->httpAccess);
74 static::assertInstanceOf(LoginManager::class, $container->loginManager); 75 static::assertInstanceOf(LoginManager::class, $container->loginManager);
76 static::assertInstanceOf(MetadataRetriever::class, $container->metadataRetriever);
75 static::assertInstanceOf(NetscapeBookmarkUtils::class, $container->netscapeBookmarkUtils); 77 static::assertInstanceOf(NetscapeBookmarkUtils::class, $container->netscapeBookmarkUtils);
76 static::assertInstanceOf(PageBuilder::class, $container->pageBuilder); 78 static::assertInstanceOf(PageBuilder::class, $container->pageBuilder);
77 static::assertInstanceOf(PageCacheManager::class, $container->pageCacheManager); 79 static::assertInstanceOf(PageCacheManager::class, $container->pageCacheManager);
diff --git a/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php b/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php
index 2eb95251..4fd88480 100644
--- a/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php
+++ b/tests/front/controller/admin/ManageShaareControllerTest/DisplayCreateFormTest.php
@@ -9,6 +9,7 @@ use Shaarli\Config\ConfigManager;
9use Shaarli\Front\Controller\Admin\FrontAdminControllerMockHelper; 9use Shaarli\Front\Controller\Admin\FrontAdminControllerMockHelper;
10use Shaarli\Front\Controller\Admin\ManageShaareController; 10use Shaarli\Front\Controller\Admin\ManageShaareController;
11use Shaarli\Http\HttpAccess; 11use Shaarli\Http\HttpAccess;
12use Shaarli\Http\MetadataRetriever;
12use Shaarli\TestCase; 13use Shaarli\TestCase;
13use Slim\Http\Request; 14use Slim\Http\Request;
14use Slim\Http\Response; 15use Slim\Http\Response;
@@ -25,6 +26,7 @@ class DisplayCreateFormTest extends TestCase
25 $this->createContainer(); 26 $this->createContainer();
26 27
27 $this->container->httpAccess = $this->createMock(HttpAccess::class); 28 $this->container->httpAccess = $this->createMock(HttpAccess::class);
29 $this->container->metadataRetriever = $this->createMock(MetadataRetriever::class);
28 $this->controller = new ManageShaareController($this->container); 30 $this->controller = new ManageShaareController($this->container);
29 } 31 }
30 32
@@ -32,7 +34,7 @@ class DisplayCreateFormTest extends TestCase
32 * Test displaying bookmark create form 34 * Test displaying bookmark create form
33 * Ensure that every step of the standard workflow works properly. 35 * Ensure that every step of the standard workflow works properly.
34 */ 36 */
35 public function testDisplayCreateFormWithUrl(): void 37 public function testDisplayCreateFormWithUrlAndWithMetadataRetrieval(): void
36 { 38 {
37 $this->container->environment = [ 39 $this->container->environment = [
38 'HTTP_REFERER' => $referer = 'http://shaarli/subfolder/controller/?searchtag=abc' 40 'HTTP_REFERER' => $referer = 'http://shaarli/subfolder/controller/?searchtag=abc'
@@ -53,40 +55,20 @@ class DisplayCreateFormTest extends TestCase
53 }); 55 });
54 $response = new Response(); 56 $response = new Response();
55 57
56 $this->container->httpAccess 58 $this->container->conf = $this->createMock(ConfigManager::class);
57 ->expects(static::once()) 59 $this->container->conf->method('get')->willReturnCallback(function (string $param, $default) {
58 ->method('getCurlDownloadCallback') 60 if ($param === 'general.enable_async_metadata') {
59 ->willReturnCallback( 61 return false;
60 function (&$charset, &$title, &$description, &$tags) use ( 62 }
61 $remoteTitle, 63
62 $remoteDesc, 64 return $default;
63 $remoteTags 65 });
64 ): callable { 66
65 return function () use ( 67 $this->container->metadataRetriever->expects(static::once())->method('retrieve')->willReturn([
66 &$charset, 68 'title' => $remoteTitle,
67 &$title, 69 'description' => $remoteDesc,
68 &$description, 70 'tags' => $remoteTags,
69 &$tags, 71 ]);
70 $remoteTitle,
71 $remoteDesc,
72 $remoteTags
73 ): void {
74 $charset = 'ISO-8859-1';
75 $title = $remoteTitle;
76 $description = $remoteDesc;
77 $tags = $remoteTags;
78 };
79 }
80 )
81 ;
82 $this->container->httpAccess
83 ->expects(static::once())
84 ->method('getHttpResponse')
85 ->with($expectedUrl, 30, 4194304)
86 ->willReturnCallback(function($url, $timeout, $maxBytes, $callback): void {
87 $callback();
88 })
89 ;
90 72
91 $this->container->bookmarkService 73 $this->container->bookmarkService
92 ->expects(static::once()) 74 ->expects(static::once())
@@ -127,6 +109,72 @@ class DisplayCreateFormTest extends TestCase
127 static::assertSame($tags, $assignedVariables['tags']); 109 static::assertSame($tags, $assignedVariables['tags']);
128 static::assertArrayHasKey('source', $assignedVariables); 110 static::assertArrayHasKey('source', $assignedVariables);
129 static::assertArrayHasKey('default_private_links', $assignedVariables); 111 static::assertArrayHasKey('default_private_links', $assignedVariables);
112 static::assertArrayHasKey('async_metadata', $assignedVariables);
113 static::assertArrayHasKey('retrieve_description', $assignedVariables);
114 }
115
116 /**
117 * Test displaying bookmark create form without any external metadata retrieval attempt
118 */
119 public function testDisplayCreateFormWithUrlAndWithoutMetadata(): void
120 {
121 $this->container->environment = [
122 'HTTP_REFERER' => $referer = 'http://shaarli/subfolder/controller/?searchtag=abc'
123 ];
124
125 $assignedVariables = [];
126 $this->assignTemplateVars($assignedVariables);
127
128 $url = 'http://url.tld/other?part=3&utm_ad=pay#hash';
129 $expectedUrl = str_replace('&utm_ad=pay', '', $url);
130
131 $request = $this->createMock(Request::class);
132 $request->method('getParam')->willReturnCallback(function (string $key) use ($url): ?string {
133 return $key === 'post' ? $url : null;
134 });
135 $response = new Response();
136
137 $this->container->metadataRetriever->expects(static::never())->method('retrieve');
138
139 $this->container->bookmarkService
140 ->expects(static::once())
141 ->method('bookmarksCountPerTag')
142 ->willReturn($tags = ['tag1' => 2, 'tag2' => 1])
143 ;
144
145 // Make sure that PluginManager hook is triggered
146 $this->container->pluginManager
147 ->expects(static::at(0))
148 ->method('executeHooks')
149 ->willReturnCallback(function (string $hook, array $data): array {
150 static::assertSame('render_editlink', $hook);
151 static::assertSame('', $data['link']['title']);
152 static::assertSame('', $data['link']['description']);
153
154 return $data;
155 })
156 ;
157
158 $result = $this->controller->displayCreateForm($request, $response);
159
160 static::assertSame(200, $result->getStatusCode());
161 static::assertSame('editlink', (string) $result->getBody());
162
163 static::assertSame('Shaare - Shaarli', $assignedVariables['pagetitle']);
164
165 static::assertSame($expectedUrl, $assignedVariables['link']['url']);
166 static::assertSame('', $assignedVariables['link']['title']);
167 static::assertSame('', $assignedVariables['link']['description']);
168 static::assertSame('', $assignedVariables['link']['tags']);
169 static::assertFalse($assignedVariables['link']['private']);
170
171 static::assertTrue($assignedVariables['link_is_new']);
172 static::assertSame($referer, $assignedVariables['http_referer']);
173 static::assertSame($tags, $assignedVariables['tags']);
174 static::assertArrayHasKey('source', $assignedVariables);
175 static::assertArrayHasKey('default_private_links', $assignedVariables);
176 static::assertArrayHasKey('async_metadata', $assignedVariables);
177 static::assertArrayHasKey('retrieve_description', $assignedVariables);
130 } 178 }
131 179
132 /** 180 /**
diff --git a/tests/http/MetadataRetrieverTest.php b/tests/http/MetadataRetrieverTest.php
new file mode 100644
index 00000000..3c9eaa0e
--- /dev/null
+++ b/tests/http/MetadataRetrieverTest.php
@@ -0,0 +1,154 @@
1<?php
2
3declare(strict_types=1);
4
5namespace Shaarli\Http;
6
7use PHPUnit\Framework\TestCase;
8use Shaarli\Config\ConfigManager;
9
10class MetadataRetrieverTest extends TestCase
11{
12 /** @var MetadataRetriever */
13 protected $retriever;
14
15 /** @var ConfigManager */
16 protected $conf;
17
18 /** @var HttpAccess */
19 protected $httpAccess;
20
21 public function setUp(): void
22 {
23 $this->conf = $this->createMock(ConfigManager::class);
24 $this->httpAccess = $this->createMock(HttpAccess::class);
25 $this->retriever = new MetadataRetriever($this->conf, $this->httpAccess);
26
27 $this->conf->method('get')->willReturnCallback(function (string $param, $default) {
28 return $default === null ? $param : $default;
29 });
30 }
31
32 /**
33 * Test metadata retrieve() with values returned
34 */
35 public function testFullRetrieval(): void
36 {
37 $url = 'https://domain.tld/link';
38 $remoteTitle = 'Remote Title ';
39 $remoteDesc = 'Sometimes the meta description is relevant.';
40 $remoteTags = 'abc def';
41 $remoteCharset = 'utf-8';
42
43 $expectedResult = [
44 'title' => $remoteTitle,
45 'description' => $remoteDesc,
46 'tags' => $remoteTags,
47 ];
48
49 $this->httpAccess
50 ->expects(static::once())
51 ->method('getCurlHeaderCallback')
52 ->willReturnCallback(
53 function (&$charset) use (
54 $remoteCharset
55 ): callable {
56 return function () use (
57 &$charset,
58 $remoteCharset
59 ): void {
60 $charset = $remoteCharset;
61 };
62 }
63 )
64 ;
65 $this->httpAccess
66 ->expects(static::once())
67 ->method('getCurlDownloadCallback')
68 ->willReturnCallback(
69 function (&$charset, &$title, &$description, &$tags) use (
70 $remoteCharset,
71 $remoteTitle,
72 $remoteDesc,
73 $remoteTags
74 ): callable {
75 return function () use (
76 &$charset,
77 &$title,
78 &$description,
79 &$tags,
80 $remoteCharset,
81 $remoteTitle,
82 $remoteDesc,
83 $remoteTags
84 ): void {
85 static::assertSame($remoteCharset, $charset);
86
87 $title = $remoteTitle;
88 $description = $remoteDesc;
89 $tags = $remoteTags;
90 };
91 }
92 )
93 ;
94 $this->httpAccess
95 ->expects(static::once())
96 ->method('getHttpResponse')
97 ->with($url, 30, 4194304)
98 ->willReturnCallback(function($url, $timeout, $maxBytes, $headerCallback, $dlCallback): void {
99 $headerCallback();
100 $dlCallback();
101 })
102 ;
103
104 $result = $this->retriever->retrieve($url);
105
106 static::assertSame($expectedResult, $result);
107 }
108
109 /**
110 * Test metadata retrieve() without any value
111 */
112 public function testEmptyRetrieval(): void
113 {
114 $url = 'https://domain.tld/link';
115
116 $expectedResult = [
117 'title' => null,
118 'description' => null,
119 'tags' => null,
120 ];
121
122 $this->httpAccess
123 ->expects(static::once())
124 ->method('getCurlDownloadCallback')
125 ->willReturnCallback(
126 function (): callable {
127 return function (): void {};
128 }
129 )
130 ;
131 $this->httpAccess
132 ->expects(static::once())
133 ->method('getCurlHeaderCallback')
134 ->willReturnCallback(
135 function (): callable {
136 return function (): void {};
137 }
138 )
139 ;
140 $this->httpAccess
141 ->expects(static::once())
142 ->method('getHttpResponse')
143 ->with($url, 30, 4194304)
144 ->willReturnCallback(function($url, $timeout, $maxBytes, $headerCallback, $dlCallback): void {
145 $headerCallback();
146 $dlCallback();
147 })
148 ;
149
150 $result = $this->retriever->retrieve($url);
151
152 static::assertSame($expectedResult, $result);
153 }
154}
diff --git a/tpl/default/editlink.html b/tpl/default/editlink.html
index 568545bd..7ab7e1fe 100644
--- a/tpl/default/editlink.html
+++ b/tpl/default/editlink.html
@@ -12,6 +12,8 @@
12 action="{$base_path}/admin/shaare" 12 action="{$base_path}/admin/shaare"
13 class="page-form pure-u-lg-3-5 pure-u-22-24 page-form page-form-light" 13 class="page-form pure-u-lg-3-5 pure-u-22-24 page-form page-form-light"
14 > 14 >
15 {$asyncLoadClass=$link_is_new && $async_metadata && empty($link.title) ? 'loading-input' : ''}
16
15 <h2 class="window-title"> 17 <h2 class="window-title">
16 {if="!$link_is_new"}{'Edit Shaare'|t}{else}{'New Shaare'|t}{/if} 18 {if="!$link_is_new"}{'Edit Shaare'|t}{else}{'New Shaare'|t}{/if}
17 </h2> 19 </h2>
@@ -28,21 +30,32 @@
28 <div> 30 <div>
29 <label for="lf_title">{'Title'|t}</label> 31 <label for="lf_title">{'Title'|t}</label>
30 </div> 32 </div>
31 <div> 33 <div class="{$asyncLoadClass}">
32 <input type="text" name="lf_title" id="lf_title" value="{$link.title}" class="lf_input autofocus"> 34 <input type="text" name="lf_title" id="lf_title" value="{$link.title}"
35 class="lf_input {if="!$async_metadata"}autofocus{/if}"
36 >
37 <div class="icon-container">
38 <i class="loader"></i>
39 </div>
33 </div> 40 </div>
34 <div> 41 <div>
35 <label for="lf_description">{'Description'|t}</label> 42 <label for="lf_description">{'Description'|t}</label>
36 </div> 43 </div>
37 <div> 44 <div class="{if="$retrieve_description"}{$asyncLoadClass}{/if}">
38 <textarea name="lf_description" id="lf_description" class="autofocus">{$link.description}</textarea> 45 <textarea name="lf_description" id="lf_description" class="autofocus">{$link.description}</textarea>
46 <div class="icon-container">
47 <i class="loader"></i>
48 </div>
39 </div> 49 </div>
40 <div> 50 <div>
41 <label for="lf_tags">{'Tags'|t}</label> 51 <label for="lf_tags">{'Tags'|t}</label>
42 </div> 52 </div>
43 <div> 53 <div class="{if="$retrieve_description"}{$asyncLoadClass}{/if}">
44 <input type="text" name="lf_tags" id="lf_tags" value="{$link.tags}" class="lf_input autofocus" 54 <input type="text" name="lf_tags" id="lf_tags" value="{$link.tags}" class="lf_input autofocus"
45 data-list="{loop="$tags"}{$key}, {/loop}" data-multiple data-autofirst autocomplete="off" > 55 data-list="{loop="$tags"}{$key}, {/loop}" data-multiple data-autofirst autocomplete="off" >
56 <div class="icon-container">
57 <i class="loader"></i>
58 </div>
46 </div> 59 </div>
47 60
48 <div> 61 <div>
@@ -88,5 +101,6 @@
88 </form> 101 </form>
89 </div> 102 </div>
90 {include="page.footer"} 103 {include="page.footer"}
104 {if="$link_is_new && $async_metadata"}<script src="{$asset_path}/js/metadata.min.js?v={$version_hash}#"></script>{/if}
91</body> 105</body>
92</html> 106</html>
diff --git a/webpack.config.js b/webpack.config.js
index a73758cc..8e3d1470 100644
--- a/webpack.config.js
+++ b/webpack.config.js
@@ -20,6 +20,7 @@ module.exports = [
20 entry: { 20 entry: {
21 thumbnails: './assets/common/js/thumbnails.js', 21 thumbnails: './assets/common/js/thumbnails.js',
22 thumbnails_update: './assets/common/js/thumbnails-update.js', 22 thumbnails_update: './assets/common/js/thumbnails-update.js',
23 metadata: './assets/common/js/metadata.js',
23 pluginsadmin: './assets/default/js/plugins-admin.js', 24 pluginsadmin: './assets/default/js/plugins-admin.js',
24 shaarli: [ 25 shaarli: [
25 './assets/default/js/base.js', 26 './assets/default/js/base.js',
@@ -99,6 +100,7 @@ module.exports = [
99 ].concat(glob.sync('./assets/vintage/img/*')), 100 ].concat(glob.sync('./assets/vintage/img/*')),
100 markdown: './assets/common/css/markdown.css', 101 markdown: './assets/common/css/markdown.css',
101 thumbnails: './assets/common/js/thumbnails.js', 102 thumbnails: './assets/common/js/thumbnails.js',
103 metadata: './assets/common/js/metadata.js',
102 thumbnails_update: './assets/common/js/thumbnails-update.js', 104 thumbnails_update: './assets/common/js/thumbnails-update.js',
103 }, 105 },
104 output: { 106 output: {
diff --git a/yarn.lock b/yarn.lock
index 0a12820c..55bd9827 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2912,6 +2912,11 @@ hash.js@^1.0.0, hash.js@^1.0.3:
2912 inherits "^2.0.3" 2912 inherits "^2.0.3"
2913 minimalistic-assert "^1.0.1" 2913 minimalistic-assert "^1.0.1"
2914 2914
2915he@^1.2.0:
2916 version "1.2.0"
2917 resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
2918 integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
2919
2915hmac-drbg@^1.0.0: 2920hmac-drbg@^1.0.0:
2916 version "1.0.1" 2921 version "1.0.1"
2917 resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1" 2922 resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1"