diff options
Diffstat (limited to 'src/Wallabag/CoreBundle/Helper')
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/ContentProxy.php | 180 | ||||
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/CryptoProxy.php | 2 | ||||
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/DownloadImages.php | 4 | ||||
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/EntriesExport.php | 4 | ||||
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php | 2 | ||||
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/Redirect.php | 2 | ||||
-rw-r--r-- | src/Wallabag/CoreBundle/Helper/TagsAssigner.php | 4 |
7 files changed, 184 insertions, 14 deletions
diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index fe795d42..d38811a2 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php | |||
@@ -53,6 +53,7 @@ class ContentProxy | |||
53 | 53 | ||
54 | if ((empty($content) || false === $this->validateContent($content)) && false === $disableContentUpdate) { | 54 | if ((empty($content) || false === $this->validateContent($content)) && false === $disableContentUpdate) { |
55 | $fetchedContent = $this->graby->fetchContent($url); | 55 | $fetchedContent = $this->graby->fetchContent($url); |
56 | $fetchedContent['title'] = $this->sanitizeContentTitle($fetchedContent['title'], $fetchedContent['content_type']); | ||
56 | 57 | ||
57 | // when content is imported, we have information in $content | 58 | // when content is imported, we have information in $content |
58 | // in case fetching content goes bad, we'll keep the imported information instead of overriding them | 59 | // in case fetching content goes bad, we'll keep the imported information instead of overriding them |
@@ -65,6 +66,13 @@ class ContentProxy | |||
65 | // so we'll be able to refetch it in the future | 66 | // so we'll be able to refetch it in the future |
66 | $content['url'] = !empty($content['url']) ? $content['url'] : $url; | 67 | $content['url'] = !empty($content['url']) ? $content['url'] : $url; |
67 | 68 | ||
69 | // In one case (at least in tests), url is empty here | ||
70 | // so we set it using $url provided in the updateEntry call. | ||
71 | // Not sure what are the other possible cases where this property is empty | ||
72 | if (empty($entry->getUrl()) && !empty($url)) { | ||
73 | $entry->setUrl($url); | ||
74 | } | ||
75 | |||
68 | $this->stockEntry($entry, $content); | 76 | $this->stockEntry($entry, $content); |
69 | } | 77 | } |
70 | 78 | ||
@@ -85,7 +93,7 @@ class ContentProxy | |||
85 | (new LocaleConstraint()) | 93 | (new LocaleConstraint()) |
86 | ); | 94 | ); |
87 | 95 | ||
88 | if (0 === count($errors)) { | 96 | if (0 === \count($errors)) { |
89 | $entry->setLanguage($value); | 97 | $entry->setLanguage($value); |
90 | 98 | ||
91 | return; | 99 | return; |
@@ -107,7 +115,7 @@ class ContentProxy | |||
107 | (new UrlConstraint()) | 115 | (new UrlConstraint()) |
108 | ); | 116 | ); |
109 | 117 | ||
110 | if (0 === count($errors)) { | 118 | if (0 === \count($errors)) { |
111 | $entry->setPreviewPicture($value); | 119 | $entry->setPreviewPicture($value); |
112 | 120 | ||
113 | return; | 121 | return; |
@@ -177,6 +185,59 @@ class ContentProxy | |||
177 | } | 185 | } |
178 | 186 | ||
179 | /** | 187 | /** |
188 | * Try to sanitize the title of the fetched content from wrong character encodings and invalid UTF-8 character. | ||
189 | * | ||
190 | * @param $title | ||
191 | * @param $contentType | ||
192 | * | ||
193 | * @return string | ||
194 | */ | ||
195 | private function sanitizeContentTitle($title, $contentType) | ||
196 | { | ||
197 | if ('application/pdf' === $contentType) { | ||
198 | $title = $this->convertPdfEncodingToUTF8($title); | ||
199 | } | ||
200 | |||
201 | return $this->sanitizeUTF8Text($title); | ||
202 | } | ||
203 | |||
204 | /** | ||
205 | * If the title from the fetched content comes from a PDF, then its very possible that the character encoding is not | ||
206 | * UTF-8. This methods tries to identify the character encoding and translate the title to UTF-8. | ||
207 | * | ||
208 | * @param $title | ||
209 | * | ||
210 | * @return string (maybe contains invalid UTF-8 character) | ||
211 | */ | ||
212 | private function convertPdfEncodingToUTF8($title) | ||
213 | { | ||
214 | // first try UTF-8 because its easier to detect its present/absence | ||
215 | foreach (['UTF-8', 'UTF-16BE', 'WINDOWS-1252'] as $encoding) { | ||
216 | if (mb_check_encoding($title, $encoding)) { | ||
217 | return mb_convert_encoding($title, 'UTF-8', $encoding); | ||
218 | } | ||
219 | } | ||
220 | |||
221 | return $title; | ||
222 | } | ||
223 | |||
224 | /** | ||
225 | * Remove invalid UTF-8 characters from the given string. | ||
226 | * | ||
227 | * @param string $rawText | ||
228 | * | ||
229 | * @return string | ||
230 | */ | ||
231 | private function sanitizeUTF8Text($rawText) | ||
232 | { | ||
233 | if (mb_check_encoding($rawText, 'UTF-8')) { | ||
234 | return $rawText; | ||
235 | } | ||
236 | |||
237 | return iconv('UTF-8', 'UTF-8//IGNORE', $rawText); | ||
238 | } | ||
239 | |||
240 | /** | ||
180 | * Stock entry with fetched or imported content. | 241 | * Stock entry with fetched or imported content. |
181 | * Will fall back to OpenGraph data if available. | 242 | * Will fall back to OpenGraph data if available. |
182 | * | 243 | * |
@@ -185,7 +246,7 @@ class ContentProxy | |||
185 | */ | 246 | */ |
186 | private function stockEntry(Entry $entry, array $content) | 247 | private function stockEntry(Entry $entry, array $content) |
187 | { | 248 | { |
188 | $entry->setUrl($content['url']); | 249 | $this->updateOriginUrl($entry, $content['url']); |
189 | 250 | ||
190 | $this->setEntryDomainName($entry); | 251 | $this->setEntryDomainName($entry); |
191 | 252 | ||
@@ -212,7 +273,7 @@ class ContentProxy | |||
212 | $entry->setHttpStatus($content['status']); | 273 | $entry->setHttpStatus($content['status']); |
213 | } | 274 | } |
214 | 275 | ||
215 | if (!empty($content['authors']) && is_array($content['authors'])) { | 276 | if (!empty($content['authors']) && \is_array($content['authors'])) { |
216 | $entry->setPublishedBy($content['authors']); | 277 | $entry->setPublishedBy($content['authors']); |
217 | } | 278 | } |
218 | 279 | ||
@@ -233,7 +294,7 @@ class ContentProxy | |||
233 | } | 294 | } |
234 | 295 | ||
235 | // if content is an image, define it as a preview too | 296 | // if content is an image, define it as a preview too |
236 | if (!empty($content['content_type']) && in_array($this->mimeGuesser->guess($content['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { | 297 | if (!empty($content['content_type']) && \in_array($this->mimeGuesser->guess($content['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) { |
237 | $this->updatePreviewPicture($entry, $content['url']); | 298 | $this->updatePreviewPicture($entry, $content['url']); |
238 | } | 299 | } |
239 | 300 | ||
@@ -252,6 +313,115 @@ class ContentProxy | |||
252 | } | 313 | } |
253 | 314 | ||
254 | /** | 315 | /** |
316 | * Update the origin_url field when a redirection occurs | ||
317 | * This field is set if it is empty and new url does not match ignore list. | ||
318 | * | ||
319 | * @param Entry $entry | ||
320 | * @param string $url | ||
321 | */ | ||
322 | private function updateOriginUrl(Entry $entry, $url) | ||
323 | { | ||
324 | if (empty($url) || $entry->getUrl() === $url) { | ||
325 | return false; | ||
326 | } | ||
327 | |||
328 | $parsed_entry_url = parse_url($entry->getUrl()); | ||
329 | $parsed_content_url = parse_url($url); | ||
330 | |||
331 | /** | ||
332 | * The following part computes the list of part changes between two | ||
333 | * parse_url arrays. | ||
334 | * | ||
335 | * As array_diff_assoc only computes changes to go from the left array | ||
336 | * to the right one, we make two differents arrays to have both | ||
337 | * directions. We merge these two arrays and sort keys before passing | ||
338 | * the result to the switch. | ||
339 | * | ||
340 | * The resulting array gives us all changing parts between the two | ||
341 | * urls: scheme, host, path, query and/or fragment. | ||
342 | */ | ||
343 | $diff_ec = array_diff_assoc($parsed_entry_url, $parsed_content_url); | ||
344 | $diff_ce = array_diff_assoc($parsed_content_url, $parsed_entry_url); | ||
345 | |||
346 | $diff = array_merge($diff_ec, $diff_ce); | ||
347 | $diff_keys = array_keys($diff); | ||
348 | sort($diff_keys); | ||
349 | |||
350 | if ($this->ignoreUrl($entry->getUrl())) { | ||
351 | $entry->setUrl($url); | ||
352 | |||
353 | return false; | ||
354 | } | ||
355 | |||
356 | /** | ||
357 | * This switch case lets us apply different behaviors according to | ||
358 | * changing parts of urls. | ||
359 | * | ||
360 | * As $diff_keys is an array, we provide arrays as cases. ['path'] means | ||
361 | * 'only the path is different between the two urls' whereas | ||
362 | * ['fragment', 'query'] means 'only fragment and query string parts are | ||
363 | * different between the two urls'. | ||
364 | * | ||
365 | * Note that values in $diff_keys are sorted. | ||
366 | */ | ||
367 | switch ($diff_keys) { | ||
368 | case ['path']: | ||
369 | if (($parsed_entry_url['path'] . '/' === $parsed_content_url['path']) // diff is trailing slash, we only replace the url of the entry | ||
370 | || ($url === urldecode($entry->getUrl()))) { // we update entry url if new url is a decoded version of it, see EntryRepository#findByUrlAndUserId | ||
371 | $entry->setUrl($url); | ||
372 | } | ||
373 | break; | ||
374 | case ['scheme']: | ||
375 | $entry->setUrl($url); | ||
376 | break; | ||
377 | case ['fragment']: | ||
378 | // noop | ||
379 | break; | ||
380 | default: | ||
381 | if (empty($entry->getOriginUrl())) { | ||
382 | $entry->setOriginUrl($entry->getUrl()); | ||
383 | } | ||
384 | $entry->setUrl($url); | ||
385 | break; | ||
386 | } | ||
387 | } | ||
388 | |||
389 | /** | ||
390 | * Check entry url against an ignore list to replace with content url. | ||
391 | * | ||
392 | * XXX: move the ignore list in the database to let users handle it | ||
393 | * | ||
394 | * @param string $url url to test | ||
395 | * | ||
396 | * @return bool true if url matches ignore list otherwise false | ||
397 | */ | ||
398 | private function ignoreUrl($url) | ||
399 | { | ||
400 | $ignored_hosts = ['feedproxy.google.com', 'feeds.reuters.com']; | ||
401 | $ignored_patterns = ['https?://www\.lemonde\.fr/tiny.*']; | ||
402 | |||
403 | $parsed_url = parse_url($url); | ||
404 | |||
405 | $filtered = array_filter($ignored_hosts, function ($var) use ($parsed_url) { | ||
406 | return $var === $parsed_url['host']; | ||
407 | }); | ||
408 | |||
409 | if ([] !== $filtered) { | ||
410 | return true; | ||
411 | } | ||
412 | |||
413 | $filtered = array_filter($ignored_patterns, function ($var) use ($url) { | ||
414 | return preg_match("`$var`i", $url); | ||
415 | }); | ||
416 | |||
417 | if ([] !== $filtered) { | ||
418 | return true; | ||
419 | } | ||
420 | |||
421 | return false; | ||
422 | } | ||
423 | |||
424 | /** | ||
255 | * Validate that the given content has at least a title, an html and a url. | 425 | * Validate that the given content has at least a title, an html and a url. |
256 | * | 426 | * |
257 | * @param array $content | 427 | * @param array $content |
diff --git a/src/Wallabag/CoreBundle/Helper/CryptoProxy.php b/src/Wallabag/CoreBundle/Helper/CryptoProxy.php index 7d8c9888..67d73915 100644 --- a/src/Wallabag/CoreBundle/Helper/CryptoProxy.php +++ b/src/Wallabag/CoreBundle/Helper/CryptoProxy.php | |||
@@ -81,6 +81,6 @@ class CryptoProxy | |||
81 | */ | 81 | */ |
82 | private function mask($value) | 82 | private function mask($value) |
83 | { | 83 | { |
84 | return strlen($value) > 0 ? $value[0] . '*****' . $value[strlen($value) - 1] : 'Empty value'; | 84 | return \strlen($value) > 0 ? $value[0] . '*****' . $value[\strlen($value) - 1] : 'Empty value'; |
85 | } | 85 | } |
86 | } | 86 | } |
diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 487a3a23..cc3dcfce 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php | |||
@@ -198,7 +198,7 @@ class DownloadImages | |||
198 | // Must be one or more digits followed by w OR x | 198 | // Must be one or more digits followed by w OR x |
199 | $pattern = "/(?:[^\"'\s]+\s*(?:\d+[wx])+)/"; | 199 | $pattern = "/(?:[^\"'\s]+\s*(?:\d+[wx])+)/"; |
200 | preg_match_all($pattern, $srcsetAttribute, $matches); | 200 | preg_match_all($pattern, $srcsetAttribute, $matches); |
201 | $srcset = call_user_func_array('array_merge', $matches); | 201 | $srcset = \call_user_func_array('array_merge', $matches); |
202 | $srcsetUrls = array_map(function ($src) { | 202 | $srcsetUrls = array_map(function ($src) { |
203 | return trim(explode(' ', $src, 2)[0]); | 203 | return trim(explode(' ', $src, 2)[0]); |
204 | }, $srcset); | 204 | }, $srcset); |
@@ -308,7 +308,7 @@ class DownloadImages | |||
308 | $this->logger->debug('DownloadImages: Checking extension (alternative)', ['ext' => $ext]); | 308 | $this->logger->debug('DownloadImages: Checking extension (alternative)', ['ext' => $ext]); |
309 | } | 309 | } |
310 | 310 | ||
311 | if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { | 311 | if (!\in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { |
312 | $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping: ' . $imagePath); | 312 | $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping: ' . $imagePath); |
313 | 313 | ||
314 | return false; | 314 | return false; |
diff --git a/src/Wallabag/CoreBundle/Helper/EntriesExport.php b/src/Wallabag/CoreBundle/Helper/EntriesExport.php index 136f66f5..cbf1037b 100644 --- a/src/Wallabag/CoreBundle/Helper/EntriesExport.php +++ b/src/Wallabag/CoreBundle/Helper/EntriesExport.php | |||
@@ -45,7 +45,7 @@ class EntriesExport | |||
45 | */ | 45 | */ |
46 | public function setEntries($entries) | 46 | public function setEntries($entries) |
47 | { | 47 | { |
48 | if (!is_array($entries)) { | 48 | if (!\is_array($entries)) { |
49 | $this->language = $entries->getLanguage(); | 49 | $this->language = $entries->getLanguage(); |
50 | $entries = [$entries]; | 50 | $entries = [$entries]; |
51 | } | 51 | } |
@@ -325,7 +325,7 @@ class EntriesExport | |||
325 | { | 325 | { |
326 | $delimiter = ';'; | 326 | $delimiter = ';'; |
327 | $enclosure = '"'; | 327 | $enclosure = '"'; |
328 | $handle = fopen('php://memory', 'rb+'); | 328 | $handle = fopen('php://memory', 'b+r'); |
329 | 329 | ||
330 | fputcsv($handle, ['Title', 'URL', 'Content', 'Tags', 'MIME Type', 'Language', 'Creation date'], $delimiter, $enclosure); | 330 | fputcsv($handle, ['Title', 'URL', 'Content', 'Tags', 'MIME Type', 'Language', 'Creation date'], $delimiter, $enclosure); |
331 | 331 | ||
diff --git a/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php b/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php index 49c1ea41..1c2c5093 100644 --- a/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php +++ b/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php | |||
@@ -31,7 +31,7 @@ class PreparePagerForEntries | |||
31 | $user = $this->tokenStorage->getToken() ? $this->tokenStorage->getToken()->getUser() : null; | 31 | $user = $this->tokenStorage->getToken() ? $this->tokenStorage->getToken()->getUser() : null; |
32 | } | 32 | } |
33 | 33 | ||
34 | if (null === $user || !is_object($user)) { | 34 | if (null === $user || !\is_object($user)) { |
35 | return; | 35 | return; |
36 | } | 36 | } |
37 | 37 | ||
diff --git a/src/Wallabag/CoreBundle/Helper/Redirect.php b/src/Wallabag/CoreBundle/Helper/Redirect.php index abc84d08..9d1a6345 100644 --- a/src/Wallabag/CoreBundle/Helper/Redirect.php +++ b/src/Wallabag/CoreBundle/Helper/Redirect.php | |||
@@ -31,7 +31,7 @@ class Redirect | |||
31 | { | 31 | { |
32 | $user = $this->tokenStorage->getToken() ? $this->tokenStorage->getToken()->getUser() : null; | 32 | $user = $this->tokenStorage->getToken() ? $this->tokenStorage->getToken()->getUser() : null; |
33 | 33 | ||
34 | if (null === $user || !is_object($user)) { | 34 | if (null === $user || !\is_object($user)) { |
35 | return $url; | 35 | return $url; |
36 | } | 36 | } |
37 | 37 | ||
diff --git a/src/Wallabag/CoreBundle/Helper/TagsAssigner.php b/src/Wallabag/CoreBundle/Helper/TagsAssigner.php index 0bfe5c57..e6b4989f 100644 --- a/src/Wallabag/CoreBundle/Helper/TagsAssigner.php +++ b/src/Wallabag/CoreBundle/Helper/TagsAssigner.php | |||
@@ -32,7 +32,7 @@ class TagsAssigner | |||
32 | { | 32 | { |
33 | $tagsEntities = []; | 33 | $tagsEntities = []; |
34 | 34 | ||
35 | if (!is_array($tags)) { | 35 | if (!\is_array($tags)) { |
36 | $tags = explode(',', $tags); | 36 | $tags = explode(',', $tags); |
37 | } | 37 | } |
38 | 38 | ||
@@ -48,7 +48,7 @@ class TagsAssigner | |||
48 | $label = trim(mb_convert_case($label, MB_CASE_LOWER)); | 48 | $label = trim(mb_convert_case($label, MB_CASE_LOWER)); |
49 | 49 | ||
50 | // avoid empty tag | 50 | // avoid empty tag |
51 | if (0 === strlen($label)) { | 51 | if (0 === \strlen($label)) { |
52 | continue; | 52 | continue; |
53 | } | 53 | } |
54 | 54 | ||