From 8ca858ee7301ea332baa9ad9543bb72d10b5caa2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 18 Mar 2019 06:23:41 +0100 Subject: Fix PHP warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Looks like sometimes (usually from import) the `html` key isn’t available. --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index d38811a2..31953f12 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -256,18 +256,17 @@ class ContentProxy $entry->setTitle($content['open_graph']['og_title']); } - $html = $content['html']; - if (false === $html) { - $html = $this->fetchingErrorMessage; + if (empty($content['html'])) { + $content['html'] = $this->fetchingErrorMessage; if (!empty($content['open_graph']['og_description'])) { - $html .= '

But we found a short description:

'; - $html .= $content['open_graph']['og_description']; + $content['html'] .= '

But we found a short description:

'; + $content['html'] .= $content['open_graph']['og_description']; } } - $entry->setContent($html); - $entry->setReadingTime(Utils::getReadingTime($html)); + $entry->setContent($content['html']); + $entry->setReadingTime(Utils::getReadingTime($content['html'])); if (!empty($content['status'])) { $entry->setHttpStatus($content['status']); -- cgit v1.2.3 From 2dbb5b2307ceefc92b465a7cbd2d0ecf512a491b Mon Sep 17 00:00:00 2001 From: Kevin Decherf Date: Wed, 1 May 2019 14:05:38 +0200 Subject: Enable no-referrer on img tags, enable strict-origin-when-cross-origin by default Fixes #3889 Signed-off-by: Kevin Decherf --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 1 + 1 file changed, 1 insertion(+) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 31953f12..bc257ffb 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -47,6 +47,7 @@ class ContentProxy */ public function updateEntry(Entry $entry, $url, array $content = [], $disableContentUpdate = false) { + $this->graby->toggleImgNoReferrer(true); if (!empty($content['html'])) { $content['html'] = $this->graby->cleanupHtml($content['html'], $url); } -- cgit v1.2.3 From 423efadefc2459c7b4a2eabc32edaed918e1075d Mon Sep 17 00:00:00 2001 From: nicofrand Date: Fri, 10 May 2019 23:01:07 +0200 Subject: Set first picture as preview picture --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index bc257ffb..ca01dec8 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -12,8 +12,8 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Tools\Utils; /** - * This kind of proxy class take care of getting the content from an url - * and update the entry with what it found. + * This kind of proxy class takes care of getting the content from an url + * and updates the entry with what it found. */ class ContentProxy { @@ -289,13 +289,25 @@ class ContentProxy $this->updateLanguage($entry, $content['language']); } + $previewPictureUrl = ''; if (!empty($content['open_graph']['og_image'])) { - $this->updatePreviewPicture($entry, $content['open_graph']['og_image']); + $previewPictureUrl = $content['open_graph']['og_image']; } // if content is an image, define it as a preview too if (!empty($content['content_type']) && \in_array($this->mimeGuesser->guess($content['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { - $this->updatePreviewPicture($entry, $content['url']); + $previewPictureUrl = $content['url']; + } elseif (empty($previewPictureUrl)) { + $this->logger->debug('Extracting images from content to provide a default preview picture'); + $imagesUrls = DownloadImages::extractImagesUrlsFromHtml($content['html']); + $this->logger->debug(\count($imagesUrls) . ' pictures found'); + if (!empty($imagesUrls)) { + $previewPictureUrl = $imagesUrls[0]; + } + } + + if (!empty($previewPictureUrl)) { + $this->updatePreviewPicture($entry, $previewPictureUrl); } if (!empty($content['content_type'])) { -- cgit v1.2.3 From 5f08426201c336f96d593954fb45b284d7e60f4a Mon Sep 17 00:00:00 2001 From: adev Date: Sat, 11 Nov 2017 20:04:15 +0100 Subject: Fix because of some breaking changes of Graby 2.0 --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index ca01dec8..ac27e50a 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -253,16 +253,14 @@ class ContentProxy if (!empty($content['title'])) { $entry->setTitle($content['title']); - } elseif (!empty($content['open_graph']['og_title'])) { - $entry->setTitle($content['open_graph']['og_title']); } if (empty($content['html'])) { $content['html'] = $this->fetchingErrorMessage; - if (!empty($content['open_graph']['og_description'])) { + if (!empty($content['description'])) { $content['html'] .= '

But we found a short description:

'; - $content['html'] .= $content['open_graph']['og_description']; + $content['html'] .= $content['description']; } } @@ -277,8 +275,8 @@ class ContentProxy $entry->setPublishedBy($content['authors']); } - if (!empty($content['all_headers']) && $this->storeArticleHeaders) { - $entry->setHeaders($content['all_headers']); + if (!empty($content['headers'])) { + $entry->setHeaders($content['headers']); } if (!empty($content['date'])) { @@ -290,12 +288,12 @@ class ContentProxy } $previewPictureUrl = ''; - if (!empty($content['open_graph']['og_image'])) { - $previewPictureUrl = $content['open_graph']['og_image']; + if (!empty($content['image'])) { + $previewPictureUrl = $content['image']; } // if content is an image, define it as a preview too - if (!empty($content['content_type']) && \in_array($this->mimeGuesser->guess($content['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { + if (!empty($content['headers']['content_type']) && \in_array($this->mimeGuesser->guess($content['headers']['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { $previewPictureUrl = $content['url']; } elseif (empty($previewPictureUrl)) { $this->logger->debug('Extracting images from content to provide a default preview picture'); @@ -310,8 +308,8 @@ class ContentProxy $this->updatePreviewPicture($entry, $previewPictureUrl); } - if (!empty($content['content_type'])) { - $entry->setMimetype($content['content_type']); + if (!empty($content['headers']['content-type'])) { + $entry->setMimetype($content['headers']['content-type']); } try { -- cgit v1.2.3 From b6c1e1bacc59ba761d1b47ac6611d1db800f7252 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Thu, 7 Feb 2019 17:56:05 +0100 Subject: Fix some tests --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index ac27e50a..59465ad1 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -54,7 +54,11 @@ class ContentProxy if ((empty($content) || false === $this->validateContent($content)) && false === $disableContentUpdate) { $fetchedContent = $this->graby->fetchContent($url); - $fetchedContent['title'] = $this->sanitizeContentTitle($fetchedContent['title'], $fetchedContent['content_type']); + + $fetchedContent['title'] = $this->sanitizeContentTitle( + $fetchedContent['title'], + isset($fetchedContent['headers']['content-type']) ? $fetchedContent['headers']['content-type'] : '' + ); // when content is imported, we have information in $content // in case fetching content goes bad, we'll keep the imported information instead of overriding them @@ -188,8 +192,8 @@ class ContentProxy /** * Try to sanitize the title of the fetched content from wrong character encodings and invalid UTF-8 character. * - * @param $title - * @param $contentType + * @param string $title + * @param string $contentType * * @return string */ @@ -293,12 +297,15 @@ class ContentProxy } // if content is an image, define it as a preview too - if (!empty($content['headers']['content_type']) && \in_array($this->mimeGuesser->guess($content['headers']['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { + if (!empty($content['headers']['content-type']) && \in_array($this->mimeGuesser->guess($content['headers']['content-type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { $previewPictureUrl = $content['url']; + + $entry->setMimetype($content['headers']['content-type']); } elseif (empty($previewPictureUrl)) { $this->logger->debug('Extracting images from content to provide a default preview picture'); $imagesUrls = DownloadImages::extractImagesUrlsFromHtml($content['html']); $this->logger->debug(\count($imagesUrls) . ' pictures found'); + if (!empty($imagesUrls)) { $previewPictureUrl = $imagesUrls[0]; } @@ -308,10 +315,6 @@ class ContentProxy $this->updatePreviewPicture($entry, $previewPictureUrl); } - if (!empty($content['headers']['content-type'])) { - $entry->setMimetype($content['headers']['content-type']); - } - try { $this->tagger->tag($entry); } catch (\Exception $e) { -- cgit v1.2.3 From 6e68417f0356d0045d7a2aa3832507d362ddcfe8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 28 May 2019 12:02:17 +0200 Subject: Fix tests after rebase --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 59465ad1..c6fa0d98 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -299,8 +299,6 @@ class ContentProxy // if content is an image, define it as a preview too if (!empty($content['headers']['content-type']) && \in_array($this->mimeGuesser->guess($content['headers']['content-type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { $previewPictureUrl = $content['url']; - - $entry->setMimetype($content['headers']['content-type']); } elseif (empty($previewPictureUrl)) { $this->logger->debug('Extracting images from content to provide a default preview picture'); $imagesUrls = DownloadImages::extractImagesUrlsFromHtml($content['html']); @@ -311,6 +309,10 @@ class ContentProxy } } + if (!empty($content['headers']['content-type'])) { + $entry->setMimetype($content['headers']['content-type']); + } + if (!empty($previewPictureUrl)) { $this->updatePreviewPicture($entry, $previewPictureUrl); } -- cgit v1.2.3 From b7fa51ae7dd5fef2d9459100c88479413ddd3fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Mon, 10 Jul 2017 21:32:25 +0200 Subject: Added given_url in entry table - Added index on entry table for given_url field - Fix tests: The previous `bit.ly` url redirected to doc.wallabag but that url doesn't exist in the fixtures. I used our own internal "redirector" to create a redirect to an url which exist in the fixtures. Also, updating current migration to use the new `WallabagMigration`. --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 1 + 1 file changed, 1 insertion(+) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index c6fa0d98..0d6a412d 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -76,6 +76,7 @@ class ContentProxy // Not sure what are the other possible cases where this property is empty if (empty($entry->getUrl()) && !empty($url)) { $entry->setUrl($url); + $entry->setGivenUrl($url); } $this->stockEntry($entry, $content); -- cgit v1.2.3 From f3bfb875e94021a93e24a41fbc0f8d86d4dee378 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 29 May 2019 14:18:04 +0200 Subject: Use hash given url to avoid duplicate Using hashed url we can ensure an index on them to ensure it's fast. --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 0d6a412d..5901df8b 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -76,9 +76,10 @@ class ContentProxy // Not sure what are the other possible cases where this property is empty if (empty($entry->getUrl()) && !empty($url)) { $entry->setUrl($url); - $entry->setGivenUrl($url); } + $entry->setGivenUrl($url); + $this->stockEntry($entry, $content); } -- cgit v1.2.3 From 8d4ed0df0633f43fc2d65fef72c36070113844d1 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 12 Nov 2019 14:18:58 +0100 Subject: Update deps Also CS (because cs-fixer got an update) Package operations: 0 installs, 26 updates, 0 removals - Updating twig/twig (v2.12.1 => v2.12.2) - Updating symfony/symfony (v3.4.33 => v3.4.34) - Updating doctrine/event-manager (v1.0.0 => 1.1.0) - Updating doctrine/collections (v1.6.2 => 1.6.3) - Updating doctrine/cache (v1.8.1 => 1.9.0) - Updating doctrine/persistence (1.1.1 => 1.2.0) - Updating doctrine/inflector (v1.3.0 => 1.3.1) - Updating symfony/mime (v4.3.5 => v4.3.7) - Updating swiftmailer/swiftmailer (v6.2.1 => v6.2.3) - Updating symfony/swiftmailer-bundle (v3.3.0 => v3.3.1) - Updating doctrine/dbal (v2.9.2 => v2.9.3) - Updating doctrine/instantiator (1.2.0 => 1.3.0) - Updating j0k3r/graby-site-config (1.0.93 => 1.0.94) - Updating phpoption/phpoption (1.5.0 => 1.5.2) - Updating symfony/http-client-contracts (v1.1.7 => v1.1.8) - Updating symfony/http-client (v4.3.5 => v4.3.7) - Updating sensiolabs/security-checker (v6.0.2 => v6.0.3) - Updating paragonie/constant_time_encoding (v2.2.3 => v2.3.0) - Updating scheb/two-factor-bundle (v4.7.1 => v4.8.0) - Updating symfony/phpunit-bridge (v4.3.6 => v4.3.7) - Updating composer/xdebug-handler (1.3.3 => 1.4.0) - Updating friendsofphp/php-cs-fixer (v2.15.3 => v2.16.0) - Updating doctrine/data-fixtures (v1.3.2 => 1.3.3) - Updating nette/schema (v1.0.0 => v1.0.1) - Updating nikic/php-parser (v4.2.4 => v4.3.0) - Updating sentry/sentry (2.2.2 => 2.2.4) --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'src/Wallabag/CoreBundle/Helper/ContentProxy.php') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 5901df8b..9c6fa8db 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -86,7 +86,6 @@ class ContentProxy /** * Use a Symfony validator to ensure the language is well formatted. * - * @param Entry $entry * @param string $value Language to validate and save */ public function updateLanguage(Entry $entry, $value) @@ -112,7 +111,6 @@ class ContentProxy /** * Use a Symfony validator to ensure the preview picture is a real url. * - * @param Entry $entry * @param string $value URL to validate and save */ public function updatePreviewPicture(Entry $entry, $value) @@ -134,7 +132,6 @@ class ContentProxy /** * Update date. * - * @param Entry $entry * @param string $value Date to validate and save */ public function updatePublishedAt(Entry $entry, $value) @@ -161,8 +158,6 @@ class ContentProxy /** * Helper to extract and save host from entry url. - * - * @param Entry $entry */ public function setEntryDomainName(Entry $entry) { @@ -176,8 +171,6 @@ class ContentProxy * Helper to set a default title using: * - url basename, if applicable * - hostname. - * - * @param Entry $entry */ public function setDefaultEntryTitle(Entry $entry) { @@ -333,7 +326,6 @@ class ContentProxy * Update the origin_url field when a redirection occurs * This field is set if it is empty and new url does not match ignore list. * - * @param Entry $entry * @param string $url */ private function updateOriginUrl(Entry $entry, $url) @@ -441,8 +433,6 @@ class ContentProxy /** * Validate that the given content has at least a title, an html and a url. * - * @param array $content - * * @return bool true if valid otherwise false */ private function validateContent(array $content) -- cgit v1.2.3