12 files changed, 177 insertions, 133 deletions

diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index 1d4a6e27..cb46db09 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -2,35 +2,39 @@
 
 namespace Wallabag\ImportBundle\Import;
 
+use Doctrine\ORM\EntityManager;
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
-use Doctrine\ORM\EntityManager;
-use Wallabag\CoreBundle\Helper\ContentProxy;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Entity\Tag;
-use Wallabag\UserBundle\Entity\User;
-use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
-use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use Wallabag\CoreBundle\Event\EntrySavedEvent;
+use Wallabag\CoreBundle\Helper\ContentProxy;
+use Wallabag\CoreBundle\Helper\TagsAssigner;
+use Wallabag\UserBundle\Entity\User;
 
 abstract class AbstractImport implements ImportInterface
 {
     protected $em;
     protected $logger;
     protected $contentProxy;
+    protected $tagsAssigner;
     protected $eventDispatcher;
     protected $producer;
     protected $user;
     protected $markAsRead;
+    protected $disableContentUpdate = false;
     protected $skippedEntries = 0;
     protected $importedEntries = 0;
     protected $queuedEntries = 0;
 
-    public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher)
+    public function __construct(EntityManager $em, ContentProxy $contentProxy, TagsAssigner $tagsAssigner, EventDispatcherInterface $eventDispatcher)
     {
         $this->em = $em;
         $this->logger = new NullLogger();
         $this->contentProxy = $contentProxy;
+        $this->tagsAssigner = $tagsAssigner;
         $this->eventDispatcher = $eventDispatcher;
     }
 
@@ -82,21 +86,55 @@ abstract class AbstractImport implements ImportInterface
     }
 
     /**
+     * Set whether articles should be fetched for updated content.
+     *
+     * @param bool $disableContentUpdate
+     */
+    public function setDisableContentUpdate($disableContentUpdate)
+    {
+        $this->disableContentUpdate = $disableContentUpdate;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSummary()
+    {
+        return [
+            'skipped' => $this->skippedEntries,
+            'imported' => $this->importedEntries,
+            'queued' => $this->queuedEntries,
+        ];
+    }
+
+    /**
+     * Parse one entry.
+     *
+     * @param array $importedEntry
+     *
+     * @return Entry
+     */
+    abstract public function parseEntry(array $importedEntry);
+
+    /**
      * Fetch content from the ContentProxy (using graby).
      * If it fails return the given entry to be saved in all case (to avoid user to loose the content).
      *
      * @param Entry  $entry   Entry to update
      * @param string $url     Url to grab content for
      * @param array  $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
-     *
-     * @return Entry
      */
     protected function fetchContent(Entry $entry, $url, array $content = [])
     {
         try {
-            return $this->contentProxy->updateEntry($entry, $url, $content);
+            $this->contentProxy->updateEntry($entry, $url, $content, $this->disableContentUpdate);
         } catch (\Exception $e) {
-            return $entry;
+            $this->logger->error('Error trying to import an entry.', [
+                'entry_url' => $url,
+                'error_msg' => $e->getMessage(),
+            ]);
         }
     }
 
@@ -179,27 +217,6 @@ abstract class AbstractImport implements ImportInterface
     }
 
     /**
-     * {@inheritdoc}
-     */
-    public function getSummary()
-    {
-        return [
-            'skipped' => $this->skippedEntries,
-            'imported' => $this->importedEntries,
-            'queued' => $this->queuedEntries,
-        ];
-    }
-
-    /**
-     * Parse one entry.
-     *
-     * @param array $importedEntry
-     *
-     * @return Entry
-     */
-    abstract public function parseEntry(array $importedEntry);
-
-    /**
      * Set current imported entry to archived / read.
      * Implementation is different accross all imports.
      *
diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php
index 8bf7d92e..78077324 100644
--- a/src/Wallabag/ImportBundle/Import/BrowserImport.php
+++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php
@@ -3,8 +3,6 @@
 namespace Wallabag\ImportBundle\Import;
 
 use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\UserBundle\Entity\User;
-use Wallabag\CoreBundle\Helper\ContentProxy;
 use Wallabag\CoreBundle\Event\EntrySavedEvent;
 
 abstract class BrowserImport extends AbstractImport
@@ -75,6 +73,80 @@ abstract class BrowserImport extends AbstractImport
     }
 
     /**
+     * {@inheritdoc}
+     */
+    public function parseEntry(array $importedEntry)
+    {
+        if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) {
+            if ($this->producer) {
+                $this->parseEntriesForProducer($importedEntry);
+
+                return;
+            }
+
+            $this->parseEntries($importedEntry);
+
+            return;
+        }
+
+        if (array_key_exists('children', $importedEntry)) {
+            if ($this->producer) {
+                $this->parseEntriesForProducer($importedEntry['children']);
+
+                return;
+            }
+
+            $this->parseEntries($importedEntry['children']);
+
+            return;
+        }
+
+        if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) {
+            return;
+        }
+
+        $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url'];
+
+        $existingEntry = $this->em
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($url, $this->user->getId());
+
+        if (false !== $existingEntry) {
+            ++$this->skippedEntries;
+
+            return;
+        }
+
+        $data = $this->prepareEntry($importedEntry);
+
+        $entry = new Entry($this->user);
+        $entry->setUrl($data['url']);
+        $entry->setTitle($data['title']);
+
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $this->fetchContent($entry, $data['url'], $data);
+
+        if (array_key_exists('tags', $data)) {
+            $this->tagsAssigner->assignTagsToEntry(
+                $entry,
+                $data['tags']
+            );
+        }
+
+        $entry->setArchived($data['is_archived']);
+
+        if (!empty($data['created_at'])) {
+            $dt = new \DateTime();
+            $entry->setCreatedAt($dt->setTimestamp($data['created_at']));
+        }
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
+
+        return $entry;
+    }
+
+    /**
      * Parse and insert all given entries.
      *
      * @param $entries
@@ -153,84 +225,12 @@ abstract class BrowserImport extends AbstractImport
     /**
      * {@inheritdoc}
      */
-    public function parseEntry(array $importedEntry)
-    {
-        if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) {
-            if ($this->producer) {
-                $this->parseEntriesForProducer($importedEntry);
-
-                return;
-            }
-
-            $this->parseEntries($importedEntry);
-
-            return;
-        }
-
-        if (array_key_exists('children', $importedEntry)) {
-            if ($this->producer) {
-                $this->parseEntriesForProducer($importedEntry['children']);
-
-                return;
-            }
-
-            $this->parseEntries($importedEntry['children']);
-
-            return;
-        }
-
-        if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) {
-            return;
-        }
-
-        $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url'];
-
-        $existingEntry = $this->em
-            ->getRepository('WallabagCoreBundle:Entry')
-            ->findByUrlAndUserId($url, $this->user->getId());
-
-        if (false !== $existingEntry) {
-            ++$this->skippedEntries;
-
-            return;
-        }
-
-        $data = $this->prepareEntry($importedEntry);
-
-        $entry = new Entry($this->user);
-        $entry->setUrl($data['url']);
-        $entry->setTitle($data['title']);
-
-        // update entry with content (in case fetching failed, the given entry will be return)
-        $entry = $this->fetchContent($entry, $data['url'], $data);
-
-        if (array_key_exists('tags', $data)) {
-            $this->contentProxy->assignTagsToEntry(
-                $entry,
-                $data['tags']
-            );
-        }
-
-        $entry->setArchived($data['is_archived']);
-
-        if (!empty($data['created_at'])) {
-            $dt = new \DateTime();
-            $entry->setCreatedAt($dt->setTimestamp($data['created_at']));
-        }
-
-        $this->em->persist($entry);
-        ++$this->importedEntries;
-
-        return $entry;
-    }
-
-    /**
-     * {@inheritdoc}
-     */
     protected function setEntryAsRead(array $importedEntry)
     {
         $importedEntry['is_archived'] = 1;
 
         return $importedEntry;
     }
+
+    abstract protected function prepareEntry(array $entry = []);
 }
diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php
index 2667890f..e3ba636a 100644
--- a/src/Wallabag/ImportBundle/Import/ChromeImport.php
+++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php
@@ -45,7 +45,7 @@ class ChromeImport extends BrowserImport
             'created_at' => substr($entry['date_added'], 0, 10),
         ];
 
-        if (array_key_exists('tags', $entry) && $entry['tags'] != '') {
+        if (array_key_exists('tags', $entry) && $entry['tags'] !== '') {
             $data['tags'] = $entry['tags'];
         }
 
diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php
index c50c69b3..c18e7e93 100644
--- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php
+++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php
@@ -45,7 +45,7 @@ class FirefoxImport extends BrowserImport
             'created_at' => substr($entry['dateAdded'], 0, 10),
         ];
 
-        if (array_key_exists('tags', $entry) && $entry['tags'] != '') {
+        if (array_key_exists('tags', $entry) && $entry['tags'] !== '') {
             $data['tags'] = $entry['tags'];
         }
 
diff --git a/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php b/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php
index a363a566..d7df0a83 100644
--- a/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php
+++ b/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php
@@ -2,8 +2,8 @@
 
 namespace Wallabag\ImportBundle\Import;
 
-use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
+use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\Reference;
 
 class ImportCompilerPass implements CompilerPassInterface
diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
index 70a53f1a..7d70154a 100644
--- a/src/Wallabag/ImportBundle/Import/InstapaperImport.php
+++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
@@ -68,6 +68,14 @@ class InstapaperImport extends AbstractImport
                 continue;
             }
 
+            // last element in the csv is the folder where the content belong
+            // BUT it can also be the status (since status = folder in Instapaper)
+            // and we don't want archive, unread & starred to become a tag
+            $tags = null;
+            if (false === in_array($data[3], ['Archive', 'Unread', 'Starred'], true)) {
+                $tags = [$data[3]];
+            }
+
             $entries[] = [
                 'url' => $data[0],
                 'title' => $data[1],
@@ -75,6 +83,7 @@ class InstapaperImport extends AbstractImport
                 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred',
                 'is_starred' => $data[3] === 'Starred',
                 'html' => false,
+                'tags' => $tags,
             ];
         }
         fclose($handle);
@@ -116,7 +125,15 @@ class InstapaperImport extends AbstractImport
         $entry->setTitle($importedEntry['title']);
 
         // update entry with content (in case fetching failed, the given entry will be return)
-        $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry);
+        $this->fetchContent($entry, $importedEntry['url'], $importedEntry);
+
+        if (!empty($importedEntry['tags'])) {
+            $this->tagsAssigner->assignTagsToEntry(
+                $entry,
+                $importedEntry['tags'],
+                $this->em->getUnitOfWork()->getScheduledEntityInsertions()
+            );
+        }
 
         $entry->setArchived($importedEntry['is_archived']);
         $entry->setStarred($importedEntry['is_starred']);
diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php
index d9865534..110b0464 100644
--- a/src/Wallabag/ImportBundle/Import/PinboardImport.php
+++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php
@@ -109,10 +109,10 @@ class PinboardImport extends AbstractImport
         $entry->setTitle($data['title']);
 
         // update entry with content (in case fetching failed, the given entry will be return)
-        $entry = $this->fetchContent($entry, $data['url'], $data);
+        $this->fetchContent($entry, $data['url'], $data);
 
         if (!empty($data['tags'])) {
-            $this->contentProxy->assignTagsToEntry(
+            $this->tagsAssigner->assignTagsToEntry(
                 $entry,
                 $data['tags'],
                 $this->em->getUnitOfWork()->getScheduledEntityInsertions()
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 33093480..7d38826b 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -5,15 +5,13 @@ namespace Wallabag\ImportBundle\Import;
 use GuzzleHttp\Client;
 use GuzzleHttp\Exception\RequestException;
 use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\CoreBundle\Helper\ContentProxy;
 
 class PocketImport extends AbstractImport
 {
+    const NB_ELEMENTS = 5000;
     private $client;
     private $accessToken;
 
-    const NB_ELEMENTS = 5000;
-
     /**
      * Only used for test purpose.
      *
@@ -177,7 +175,7 @@ class PocketImport extends AbstractImport
      */
     public function parseEntry(array $importedEntry)
     {
-        $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
+        $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] !== '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
 
         $existingEntry = $this->em
             ->getRepository('WallabagCoreBundle:Entry')
@@ -193,18 +191,18 @@ class PocketImport extends AbstractImport
         $entry->setUrl($url);
 
         // update entry with content (in case fetching failed, the given entry will be return)
-        $entry = $this->fetchContent($entry, $url);
+        $this->fetchContent($entry, $url);
 
         // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
-        $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
+        $entry->setArchived($importedEntry['status'] === 1 || $this->markAsRead);
 
         // 0 or 1 - 1 If the item is starred
-        $entry->setStarred($importedEntry['favorite'] == 1);
+        $entry->setStarred($importedEntry['favorite'] === 1);
 
         $title = 'Untitled';
-        if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
+        if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] !== '') {
             $title = $importedEntry['resolved_title'];
-        } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
+        } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] !== '') {
             $title = $importedEntry['given_title'];
         }
 
@@ -216,7 +214,7 @@ class PocketImport extends AbstractImport
         }
 
         if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
-            $this->contentProxy->assignTagsToEntry(
+            $this->tagsAssigner->assignTagsToEntry(
                 $entry,
                 array_keys($importedEntry['tags']),
                 $this->em->getUnitOfWork()->getScheduledEntityInsertions()
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
index de320d23..002b27f4 100644
--- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -109,7 +109,7 @@ class ReadabilityImport extends AbstractImport
         $entry->setTitle($data['title']);
 
         // update entry with content (in case fetching failed, the given entry will be return)
-        $entry = $this->fetchContent($entry, $data['url'], $data);
+        $this->fetchContent($entry, $data['url'], $data);
 
         $entry->setArchived($data['is_archived']);
         $entry->setStarred($data['is_starred']);
diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php
index 702da057..c64ccd64 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagImport.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php
@@ -108,10 +108,10 @@ abstract class WallabagImport extends AbstractImport
         $entry->setTitle($data['title']);
 
         // update entry with content (in case fetching failed, the given entry will be return)
-        $entry = $this->fetchContent($entry, $data['url'], $data);
+        $this->fetchContent($entry, $data['url'], $data);
 
         if (array_key_exists('tags', $data)) {
-            $this->contentProxy->assignTagsToEntry(
+            $this->tagsAssigner->assignTagsToEntry(
                 $entry,
                 $data['tags'],
                 $this->em->getUnitOfWork()->getScheduledEntityInsertions()
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
index 59e3ce02..d585d44d 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
@@ -4,6 +4,17 @@ namespace Wallabag\ImportBundle\Import;
 
 class WallabagV1Import extends WallabagImport
 {
+    protected $fetchingErrorMessage;
+    protected $fetchingErrorMessageTitle;
+
+    public function __construct($em, $contentProxy, $tagsAssigner, $eventDispatcher, $fetchingErrorMessageTitle, $fetchingErrorMessage)
+    {
+        $this->fetchingErrorMessageTitle = $fetchingErrorMessageTitle;
+        $this->fetchingErrorMessage = $fetchingErrorMessage;
+
+        parent::__construct($em, $contentProxy, $tagsAssigner, $eventDispatcher);
+    }
+
     /**
      * {@inheritdoc}
      */
@@ -43,13 +54,14 @@ class WallabagV1Import extends WallabagImport
             'created_at' => '',
         ];
 
-        // force content to be refreshed in case on bad fetch in the v1 installation
-        if (in_array($entry['title'], $this->untitled)) {
-            $data['title'] = '';
-            $data['html'] = '';
+        // In case of a bad fetch in v1, replace title and content with v2 error strings
+        // If fetching fails again, they will get this instead of the v1 strings
+        if (in_array($entry['title'], $this->untitled, true)) {
+            $data['title'] = $this->fetchingErrorMessageTitle;
+            $data['html'] = $this->fetchingErrorMessage;
         }
 
-        if (array_key_exists('tags', $entry) && $entry['tags'] != '') {
+        if (array_key_exists('tags', $entry) && $entry['tags'] !== '') {
             $data['tags'] = $entry['tags'];
         }
 
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
index d2a89d79..3e085ecf 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
@@ -36,8 +36,8 @@ class WallabagV2Import extends WallabagImport
         return [
             'html' => $entry['content'],
             'content_type' => $entry['mimetype'],
-            'is_archived' => (int) ($entry['is_archived'] || $this->markAsRead),
-            'is_starred' => false,
+            'is_archived' => (bool) ($entry['is_archived'] || $this->markAsRead),
+            'is_starred' => (bool) $entry['is_starred'],
         ] + $entry;
     }