X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=src%2FWallabag%2FCoreBundle%2FHelper%2FContentProxy.php;h=d38811a272caa13b0ed859a99ee394c64e93b831;hb=1b220426e2e8139364b4a34678a2843c2e8bccf5;hp=2628af190b3119a952394f382368c0ab914bfee9;hpb=d76a5a6d60b6ee0d1f7efd0c8a70204f821ed99e;p=github%2Fwallabag%2Fwallabag.git diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 2628af19..d38811a2 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -53,7 +53,7 @@ class ContentProxy if ((empty($content) || false === $this->validateContent($content)) && false === $disableContentUpdate) { $fetchedContent = $this->graby->fetchContent($url); - $fetchedContent['title'] = $this->sanitizeUTF8Text($fetchedContent['title']); + $fetchedContent['title'] = $this->sanitizeContentTitle($fetchedContent['title'], $fetchedContent['content_type']); // when content is imported, we have information in $content // in case fetching content goes bad, we'll keep the imported information instead of overriding them @@ -66,29 +66,14 @@ class ContentProxy // so we'll be able to refetch it in the future $content['url'] = !empty($content['url']) ? $content['url'] : $url; - $this->stockEntry($entry, $content); - } - - /** - * Remove invalid UTF-8 characters from the given string in following steps: - * - try to interpret the given string as ISO-8859-1, convert it to UTF-8 and return it (if its valid) - * - simply remove every invalid UTF-8 character and return the result (https://stackoverflow.com/a/1433665) - * @param String $rawText - * @return string - */ - private function sanitizeUTF8Text(String $rawText) { - if (mb_check_encoding($rawText, 'utf-8')) { - return $rawText; // return because its valid utf-8 text - } - - // we assume that $text is encoded in ISO-8859-1 (and not the similar Windows-1252 or other encoding) - $convertedText = utf8_encode($rawText); - if (mb_check_encoding($convertedText, 'utf-8')) { - return $convertedText; + // In one case (at least in tests), url is empty here + // so we set it using $url provided in the updateEntry call. + // Not sure what are the other possible cases where this property is empty + if (empty($entry->getUrl()) && !empty($url)) { + $entry->setUrl($url); } - // last resort: simply remove invalid UTF-8 character because $rawText can have some every exotic encoding - return iconv("UTF-8", "UTF-8//IGNORE", $rawText); + $this->stockEntry($entry, $content); } /** @@ -199,6 +184,59 @@ class ContentProxy $entry->setTitle($path); } + /** + * Try to sanitize the title of the fetched content from wrong character encodings and invalid UTF-8 character. + * + * @param $title + * @param $contentType + * + * @return string + */ + private function sanitizeContentTitle($title, $contentType) + { + if ('application/pdf' === $contentType) { + $title = $this->convertPdfEncodingToUTF8($title); + } + + return $this->sanitizeUTF8Text($title); + } + + /** + * If the title from the fetched content comes from a PDF, then its very possible that the character encoding is not + * UTF-8. This methods tries to identify the character encoding and translate the title to UTF-8. + * + * @param $title + * + * @return string (maybe contains invalid UTF-8 character) + */ + private function convertPdfEncodingToUTF8($title) + { + // first try UTF-8 because its easier to detect its present/absence + foreach (['UTF-8', 'UTF-16BE', 'WINDOWS-1252'] as $encoding) { + if (mb_check_encoding($title, $encoding)) { + return mb_convert_encoding($title, 'UTF-8', $encoding); + } + } + + return $title; + } + + /** + * Remove invalid UTF-8 characters from the given string. + * + * @param string $rawText + * + * @return string + */ + private function sanitizeUTF8Text($rawText) + { + if (mb_check_encoding($rawText, 'UTF-8')) { + return $rawText; + } + + return iconv('UTF-8', 'UTF-8//IGNORE', $rawText); + } + /** * Stock entry with fetched or imported content. * Will fall back to OpenGraph data if available. @@ -208,7 +246,7 @@ class ContentProxy */ private function stockEntry(Entry $entry, array $content) { - $entry->setUrl($content['url']); + $this->updateOriginUrl($entry, $content['url']); $this->setEntryDomainName($entry); @@ -274,6 +312,115 @@ class ContentProxy } } + /** + * Update the origin_url field when a redirection occurs + * This field is set if it is empty and new url does not match ignore list. + * + * @param Entry $entry + * @param string $url + */ + private function updateOriginUrl(Entry $entry, $url) + { + if (empty($url) || $entry->getUrl() === $url) { + return false; + } + + $parsed_entry_url = parse_url($entry->getUrl()); + $parsed_content_url = parse_url($url); + + /** + * The following part computes the list of part changes between two + * parse_url arrays. + * + * As array_diff_assoc only computes changes to go from the left array + * to the right one, we make two differents arrays to have both + * directions. We merge these two arrays and sort keys before passing + * the result to the switch. + * + * The resulting array gives us all changing parts between the two + * urls: scheme, host, path, query and/or fragment. + */ + $diff_ec = array_diff_assoc($parsed_entry_url, $parsed_content_url); + $diff_ce = array_diff_assoc($parsed_content_url, $parsed_entry_url); + + $diff = array_merge($diff_ec, $diff_ce); + $diff_keys = array_keys($diff); + sort($diff_keys); + + if ($this->ignoreUrl($entry->getUrl())) { + $entry->setUrl($url); + + return false; + } + + /** + * This switch case lets us apply different behaviors according to + * changing parts of urls. + * + * As $diff_keys is an array, we provide arrays as cases. ['path'] means + * 'only the path is different between the two urls' whereas + * ['fragment', 'query'] means 'only fragment and query string parts are + * different between the two urls'. + * + * Note that values in $diff_keys are sorted. + */ + switch ($diff_keys) { + case ['path']: + if (($parsed_entry_url['path'] . '/' === $parsed_content_url['path']) // diff is trailing slash, we only replace the url of the entry + || ($url === urldecode($entry->getUrl()))) { // we update entry url if new url is a decoded version of it, see EntryRepository#findByUrlAndUserId + $entry->setUrl($url); + } + break; + case ['scheme']: + $entry->setUrl($url); + break; + case ['fragment']: + // noop + break; + default: + if (empty($entry->getOriginUrl())) { + $entry->setOriginUrl($entry->getUrl()); + } + $entry->setUrl($url); + break; + } + } + + /** + * Check entry url against an ignore list to replace with content url. + * + * XXX: move the ignore list in the database to let users handle it + * + * @param string $url url to test + * + * @return bool true if url matches ignore list otherwise false + */ + private function ignoreUrl($url) + { + $ignored_hosts = ['feedproxy.google.com', 'feeds.reuters.com']; + $ignored_patterns = ['https?://www\.lemonde\.fr/tiny.*']; + + $parsed_url = parse_url($url); + + $filtered = array_filter($ignored_hosts, function ($var) use ($parsed_url) { + return $var === $parsed_url['host']; + }); + + if ([] !== $filtered) { + return true; + } + + $filtered = array_filter($ignored_patterns, function ($var) use ($url) { + return preg_match("`$var`i", $url); + }); + + if ([] !== $filtered) { + return true; + } + + return false; + } + /** * Validate that the given content has at least a title, an html and a url. *