7 files changed, 108 insertions, 15 deletions

diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php
index fe795d42..d4ea608f 100644
--- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php
+++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php
@@ -53,6 +53,7 @@ class ContentProxy
 
         if ((empty($content) || false === $this->validateContent($content)) && false === $disableContentUpdate) {
             $fetchedContent = $this->graby->fetchContent($url);
+            $fetchedContent['title'] = $this->sanitizeContentTitle($fetchedContent['title'], $fetchedContent['content_type']);
 
             // when content is imported, we have information in $content
             // in case fetching content goes bad, we'll keep the imported information instead of overriding them
@@ -85,7 +86,7 @@ class ContentProxy
             (new LocaleConstraint())
         );
 
-        if (0 === count($errors)) {
+        if (0 === \count($errors)) {
             $entry->setLanguage($value);
 
             return;
@@ -107,7 +108,7 @@ class ContentProxy
             (new UrlConstraint())
         );
 
-        if (0 === count($errors)) {
+        if (0 === \count($errors)) {
             $entry->setPreviewPicture($value);
 
             return;
@@ -177,6 +178,59 @@ class ContentProxy
     }
 
     /**
+     * Try to sanitize the title of the fetched content from wrong character encodings and invalid UTF-8 character.
+     *
+     * @param $title
+     * @param $contentType
+     *
+     * @return string
+     */
+    private function sanitizeContentTitle($title, $contentType)
+    {
+        if ('application/pdf' === $contentType) {
+            $title = $this->convertPdfEncodingToUTF8($title);
+        }
+
+        return $this->sanitizeUTF8Text($title);
+    }
+
+    /**
+     * If the title from the fetched content comes from a PDF, then its very possible that the character encoding is not
+     * UTF-8. This methods tries to identify the character encoding and translate the title to UTF-8.
+     *
+     * @param $title
+     *
+     * @return string (maybe contains invalid UTF-8 character)
+     */
+    private function convertPdfEncodingToUTF8($title)
+    {
+        // first try UTF-8 because its easier to detect its present/absence
+        foreach (['UTF-8', 'UTF-16BE', 'WINDOWS-1252'] as $encoding) {
+            if (mb_check_encoding($title, $encoding)) {
+                return mb_convert_encoding($title, 'UTF-8', $encoding);
+            }
+        }
+
+        return $title;
+    }
+
+    /**
+     * Remove invalid UTF-8 characters from the given string.
+     *
+     * @param string $rawText
+     *
+     * @return string
+     */
+    private function sanitizeUTF8Text($rawText)
+    {
+        if (mb_check_encoding($rawText, 'UTF-8')) {
+            return $rawText;
+        }
+
+        return iconv('UTF-8', 'UTF-8//IGNORE', $rawText);
+    }
+
+    /**
      * Stock entry with fetched or imported content.
      * Will fall back to OpenGraph data if available.
      *
@@ -212,7 +266,7 @@ class ContentProxy
             $entry->setHttpStatus($content['status']);
         }
 
-        if (!empty($content['authors']) && is_array($content['authors'])) {
+        if (!empty($content['authors']) && \is_array($content['authors'])) {
             $entry->setPublishedBy($content['authors']);
         }
 
@@ -233,7 +287,7 @@ class ContentProxy
         }
 
         // if content is an image, define it as a preview too
-        if (!empty($content['content_type']) && in_array($this->mimeGuesser->guess($content['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) {
+        if (!empty($content['content_type']) && \in_array($this->mimeGuesser->guess($content['content_type']), ['jpeg', 'jpg', 'gif', 'png'], true)) {
             $this->updatePreviewPicture($entry, $content['url']);
         }
 
diff --git a/src/Wallabag/CoreBundle/Helper/CryptoProxy.php b/src/Wallabag/CoreBundle/Helper/CryptoProxy.php
index 7d8c9888..67d73915 100644
--- a/src/Wallabag/CoreBundle/Helper/CryptoProxy.php
+++ b/src/Wallabag/CoreBundle/Helper/CryptoProxy.php
@@ -81,6 +81,6 @@ class CryptoProxy
      */
     private function mask($value)
     {
-        return strlen($value) > 0 ? $value[0] . '*****' . $value[strlen($value) - 1] : 'Empty value';
+        return \strlen($value) > 0 ? $value[0] . '*****' . $value[\strlen($value) - 1] : 'Empty value';
     }
 }
diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php
index 252ba57c..cc3dcfce 100644
--- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php
+++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php
@@ -42,14 +42,17 @@ class DownloadImages
     public function processHtml($entryId, $html, $url)
     {
         $crawler = new Crawler($html);
-        $result = $crawler
-            ->filterXpath('//img')
+        $imagesCrawler = $crawler
+            ->filterXpath('//img');
+        $imagesUrls = $imagesCrawler
             ->extract(['src']);
+        $imagesSrcsetUrls = $this->getSrcsetUrls($imagesCrawler);
+        $imagesUrls = array_unique(array_merge($imagesUrls, $imagesSrcsetUrls));
 
         $relativePath = $this->getRelativePath($entryId);
 
         // download and save the image to the folder
-        foreach ($result as $image) {
+        foreach ($imagesUrls as $image) {
             $imagePath = $this->processSingleImage($entryId, $image, $url, $relativePath);
 
             if (false === $imagePath) {
@@ -82,6 +85,10 @@ class DownloadImages
      */
     public function processSingleImage($entryId, $imagePath, $url, $relativePath = null)
     {
+        if (null === $imagePath) {
+            return false;
+        }
+
         if (null === $relativePath) {
             $relativePath = $this->getRelativePath($entryId);
         }
@@ -172,6 +179,38 @@ class DownloadImages
     }
 
     /**
+     * Get images urls from the srcset image attribute.
+     *
+     * @param Crawler $imagesCrawler
+     *
+     * @return array An array of urls
+     */
+    private function getSrcsetUrls(Crawler $imagesCrawler)
+    {
+        $urls = [];
+        $iterator = $imagesCrawler
+            ->getIterator();
+        while ($iterator->valid()) {
+            $srcsetAttribute = $iterator->current()->getAttribute('srcset');
+            if ('' !== $srcsetAttribute) {
+                // Couldn't start with " OR ' OR a white space
+                // Could be one or more white space
+                // Must be one or more digits followed by w OR x
+                $pattern = "/(?:[^\"'\s]+\s*(?:\d+[wx])+)/";
+                preg_match_all($pattern, $srcsetAttribute, $matches);
+                $srcset = \call_user_func_array('array_merge', $matches);
+                $srcsetUrls = array_map(function ($src) {
+                    return trim(explode(' ', $src, 2)[0]);
+                }, $srcset);
+                $urls = array_merge($srcsetUrls, $urls);
+            }
+            $iterator->next();
+        }
+
+        return $urls;
+    }
+
+    /**
      * Setup base folder where all images are going to be saved.
      */
     private function setFolder()
@@ -269,7 +308,7 @@ class DownloadImages
             $this->logger->debug('DownloadImages: Checking extension (alternative)', ['ext' => $ext]);
         }
 
-        if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) {
+        if (!\in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) {
             $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping: ' . $imagePath);
 
             return false;
diff --git a/src/Wallabag/CoreBundle/Helper/EntriesExport.php b/src/Wallabag/CoreBundle/Helper/EntriesExport.php
index 136f66f5..cbf1037b 100644
--- a/src/Wallabag/CoreBundle/Helper/EntriesExport.php
+++ b/src/Wallabag/CoreBundle/Helper/EntriesExport.php
@@ -45,7 +45,7 @@ class EntriesExport
      */
     public function setEntries($entries)
     {
-        if (!is_array($entries)) {
+        if (!\is_array($entries)) {
             $this->language = $entries->getLanguage();
             $entries = [$entries];
         }
@@ -325,7 +325,7 @@ class EntriesExport
     {
         $delimiter = ';';
         $enclosure = '"';
-        $handle = fopen('php://memory', 'rb+');
+        $handle = fopen('php://memory', 'b+r');
 
         fputcsv($handle, ['Title', 'URL', 'Content', 'Tags', 'MIME Type', 'Language', 'Creation date'], $delimiter, $enclosure);
 
diff --git a/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php b/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php
index 49c1ea41..1c2c5093 100644
--- a/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php
+++ b/src/Wallabag/CoreBundle/Helper/PreparePagerForEntries.php
@@ -31,7 +31,7 @@ class PreparePagerForEntries
             $user = $this->tokenStorage->getToken() ? $this->tokenStorage->getToken()->getUser() : null;
         }
 
-        if (null === $user || !is_object($user)) {
+        if (null === $user || !\is_object($user)) {
             return;
         }
 
diff --git a/src/Wallabag/CoreBundle/Helper/Redirect.php b/src/Wallabag/CoreBundle/Helper/Redirect.php
index abc84d08..9d1a6345 100644
--- a/src/Wallabag/CoreBundle/Helper/Redirect.php
+++ b/src/Wallabag/CoreBundle/Helper/Redirect.php
@@ -31,7 +31,7 @@ class Redirect
     {
         $user = $this->tokenStorage->getToken() ? $this->tokenStorage->getToken()->getUser() : null;
 
-        if (null === $user || !is_object($user)) {
+        if (null === $user || !\is_object($user)) {
             return $url;
         }
 
diff --git a/src/Wallabag/CoreBundle/Helper/TagsAssigner.php b/src/Wallabag/CoreBundle/Helper/TagsAssigner.php
index 0bfe5c57..e6b4989f 100644
--- a/src/Wallabag/CoreBundle/Helper/TagsAssigner.php
+++ b/src/Wallabag/CoreBundle/Helper/TagsAssigner.php
@@ -32,7 +32,7 @@ class TagsAssigner
     {
         $tagsEntities = [];
 
-        if (!is_array($tags)) {
+        if (!\is_array($tags)) {
             $tags = explode(',', $tags);
         }
 
@@ -48,7 +48,7 @@ class TagsAssigner
             $label = trim(mb_convert_case($label, MB_CASE_LOWER));
 
             // avoid empty tag
-            if (0 === strlen($label)) {
+            if (0 === \strlen($label)) {
                 continue;
             }