Send every imported item to the queue

Instead of queing real Entry to process, we queue all the item to import from Pocket in a raw format.
Then, the worker retrieve that information, find / create the entry and save it.

1 files changed, 112 insertions, 65 deletions

diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 7d1c0c61..27df4917 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -3,12 +3,11 @@
 namespace Wallabag\ImportBundle\Import;
 
 use OldSound\RabbitMqBundle\RabbitMq\Producer;
-use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
 use Doctrine\ORM\EntityManager;
 use GuzzleHttp\Client;
 use GuzzleHttp\Exception\RequestException;
-use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
+use Symfony\Component\Security\Core\User\UserInterface;
 use Wallabag\CoreBundle\Entity\Entry;
 use Wallabag\CoreBundle\Helper\ContentProxy;
 use Craue\ConfigBundle\Util\Config;
@@ -21,22 +20,40 @@ class PocketImport extends AbstractImport
     private $skippedEntries = 0;
     private $importedEntries = 0;
     private $markAsRead;
-    protected $accessToken;
     private $producer;
-    private $rabbitMQ;
+    protected $accessToken;
 
-    public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, Producer $producer)
+    public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
     {
-        $this->user = $tokenStorage->getToken()->getUser();
         $this->em = $em;
         $this->contentProxy = $contentProxy;
         $this->consumerKey = $craueConfig->get('pocket_consumer_key');
         $this->logger = new NullLogger();
-        $this->rabbitMQ = $craueConfig->get('rabbitmq');
+    }
+
+    /**
+     * Set RabbitMQ Producer to send each entry to a queue.
+     * This method should be called when user has enabled RabbitMQ.
+     *
+     * @param Producer $producer
+     */
+    public function setRabbitmqProducer(Producer $producer)
+    {
         $this->producer = $producer;
     }
 
     /**
+     * Set current user.
+     * Could the current *connected* user or one retrieve by the consumer.
+     *
+     * @param UserInterface $user
+     */
+    public function setUser(UserInterface $user)
+    {
+        $this->user = $user;
+    }
+
+    /**
      * {@inheritdoc}
      */
     public function getName()
@@ -168,6 +185,12 @@ class PocketImport extends AbstractImport
 
         $entries = $response->json();
 
+        if ($this->producer) {
+            $this->parseEntriesForProducer($entries['list']);
+
+            return true;
+        }
+
         $this->parseEntries($entries['list']);
 
         return true;
@@ -197,88 +220,112 @@ class PocketImport extends AbstractImport
     /**
      * @see https://getpocket.com/developer/docs/v3/retrieve
      *
-     * @param $entries
+     * @param array $entries
      */
-    private function parseEntries($entries)
+    private function parseEntries(array $entries)
     {
         $i = 1;
 
-        foreach ($entries as &$pocketEntry) {
-            $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
-
-            $existingEntry = $this->em
-                ->getRepository('WallabagCoreBundle:Entry')
-                ->findByUrlAndUserId($url, $this->user->getId());
+        foreach ($entries as $pocketEntry) {
+            $entry = $this->parseEntry($pocketEntry);
 
-            if (false !== $existingEntry) {
-                ++$this->skippedEntries;
+            if (null === $entry) {
                 continue;
             }
 
-            $entry = new Entry($this->user);
+            // flush every 20 entries
+            if (($i % 20) === 0) {
+                $this->em->flush();
+                $this->em->clear($entry);
+            }
 
-            if (!$this->rabbitMQ) {
-                $entry = $this->fetchContent($entry, $url);
+            ++$i;
+        }
 
-                // jump to next entry in case of problem while getting content
-                if (false === $entry) {
-                    ++$this->skippedEntries;
-                    continue;
-                }
-            }
+        $this->em->flush();
+    }
 
-            // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
-            if ($pocketEntry['status'] == 1 || $this->markAsRead) {
-                $entry->setArchived(true);
-            }
+    public function parseEntry(array $pocketEntry)
+    {
+        $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
 
-            // 0 or 1 - 1 If the item is starred
-            if ($pocketEntry['favorite'] == 1) {
-                $entry->setStarred(true);
-            }
+        $existingEntry = $this->em
+            ->getRepository('WallabagCoreBundle:Entry')
+            ->findByUrlAndUserId($url, $this->user->getId());
 
-            $title = 'Untitled';
-            if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
-                $title = $pocketEntry['resolved_title'];
-            } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
-                $title = $pocketEntry['given_title'];
-            }
+        if (false !== $existingEntry) {
+            ++$this->skippedEntries;
 
-            $entry->setTitle($title);
-            $entry->setUrl($url);
+            return;
+        }
 
-            // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
-            if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
-                $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
-            }
+        $entry = new Entry($this->user);
+        $entry = $this->fetchContent($entry, $url);
 
-            if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
-                $this->contentProxy->assignTagsToEntry(
-                    $entry,
-                    array_keys($pocketEntry['tags'])
-                );
-            }
+        // jump to next entry in case of problem while getting content
+        if (false === $entry) {
+            ++$this->skippedEntries;
 
-            $pocketEntry['url'] = $url;
-            $pocketEntry['userId'] = $this->user->getId();
+            return;
+        }
 
-            $this->em->persist($entry);
-            ++$this->importedEntries;
+        // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
+        if ($pocketEntry['status'] == 1 || $this->markAsRead) {
+            $entry->setArchived(true);
+        }
 
-            // flush every 20 entries
-            if (($i % 20) === 0) {
-                $this->em->flush();
-            }
+        // 0 or 1 - 1 If the item is starred
+        if ($pocketEntry['favorite'] == 1) {
+            $entry->setStarred(true);
+        }
 
-            ++$i;
+        $title = 'Untitled';
+        if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
+            $title = $pocketEntry['resolved_title'];
+        } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
+            $title = $pocketEntry['given_title'];
         }
 
-        $this->em->flush();
+        $entry->setTitle($title);
+        $entry->setUrl($url);
+
+        // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
+        if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
+            $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
+        }
+
+        if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
+            $this->contentProxy->assignTagsToEntry(
+                $entry,
+                array_keys($pocketEntry['tags'])
+            );
+        }
+
+        $this->em->persist($entry);
+        ++$this->importedEntries;
 
-        if ($this->rabbitMQ) {
-            foreach ($entries as $entry) {
-                $this->producer->publish(serialize($entry));
+        return $entry;
+    }
+
+    /**
+     * Faster parse entries for Producer.
+     * We don't care to make check at this time. They'll be done by the consumer.
+     *
+     * @param array $entries
+     */
+    public function parseEntriesForProducer($entries)
+    {
+        foreach ($entries as $pocketEntry) {
+            // set userId for the producer (it won't know which user is connected)
+            $pocketEntry['userId'] = $this->user->getId();
+
+            if ($this->markAsRead) {
+                $pocketEntry['status'] = 1;
             }
+
+            ++$this->importedEntries;
+
+            $this->producer->publish(json_encode($pocketEntry));
         }
     }
 }