X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=src%2FWallabag%2FImportBundle%2FImport%2FPocketImport.php;h=dddb87f4e561d4e075927a2f201423bf17c3916a;hb=1953a872932a63792293b4aec087880265ba89f7;hp=27df4917ae4d6b10e3041a6b574ce5aefeedd647;hpb=ef75e1220ebb76a8df019d946460ad612759f0bb;p=github%2Fwallabag%2Fwallabag.git

diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 27df4917..dddb87f4 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -2,55 +2,24 @@
 
 namespace Wallabag\ImportBundle\Import;
 
-use OldSound\RabbitMqBundle\RabbitMq\Producer;
-use Psr\Log\NullLogger;
-use Doctrine\ORM\EntityManager;
 use GuzzleHttp\Client;
 use GuzzleHttp\Exception\RequestException;
-use Symfony\Component\Security\Core\User\UserInterface;
 use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\CoreBundle\Helper\ContentProxy;
-use Craue\ConfigBundle\Util\Config;
 
 class PocketImport extends AbstractImport
 {
-    private $user;
+    const NB_ELEMENTS = 5000;
     private $client;
-    private $consumerKey;
-    private $skippedEntries = 0;
-    private $importedEntries = 0;
-    private $markAsRead;
-    private $producer;
-    protected $accessToken;
-
-    public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
-    {
-        $this->em = $em;
-        $this->contentProxy = $contentProxy;
-        $this->consumerKey = $craueConfig->get('pocket_consumer_key');
-        $this->logger = new NullLogger();
-    }
-
-    /**
-     * Set RabbitMQ Producer to send each entry to a queue.
-     * This method should be called when user has enabled RabbitMQ.
-     *
-     * @param Producer $producer
-     */
-    public function setRabbitmqProducer(Producer $producer)
-    {
-        $this->producer = $producer;
-    }
+    private $accessToken;
 
     /**
-     * Set current user.
-     * Could the current *connected* user or one retrieve by the consumer.
+     * Only used for test purpose.
      *
-     * @param UserInterface $user
+     * @return string
      */
-    public function setUser(UserInterface $user)
+    public function getAccessToken()
     {
-        $this->user = $user;
+        return $this->accessToken;
     }
 
     /**
@@ -89,7 +58,7 @@ class PocketImport extends AbstractImport
         $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
             [
                 'body' => json_encode([
-                    'consumer_key' => $this->consumerKey,
+                    'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
                     'redirect_uri' => $redirectUri,
                 ]),
             ]
@@ -119,7 +88,7 @@ class PocketImport extends AbstractImport
         $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
             [
                 'body' => json_encode([
-                    'consumer_key' => $this->consumerKey,
+                    'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
                     'code' => $code,
                 ]),
             ]
@@ -138,39 +107,23 @@ class PocketImport extends AbstractImport
         return true;
     }
 
-    /**
-     * Set whether articles must be all marked as read.
-     *
-     * @param bool $markAsRead
-     */
-    public function setMarkAsRead($markAsRead)
-    {
-        $this->markAsRead = $markAsRead;
-
-        return $this;
-    }
-
-    /**
-     * Get whether articles must be all marked as read.
-     */
-    public function getMarkAsRead()
-    {
-        return $this->markAsRead;
-    }
-
     /**
      * {@inheritdoc}
      */
-    public function import()
+    public function import($offset = 0)
     {
+        static $run = 0;
+
         $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
             [
                 'body' => json_encode([
-                    'consumer_key' => $this->consumerKey,
+                    'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
                     'access_token' => $this->accessToken,
                     'detailType' => 'complete',
                     'state' => 'all',
-                    'sort' => 'oldest',
+                    'sort' => 'newest',
+                    'count' => self::NB_ELEMENTS,
+                    'offset' => $offset,
                 ]),
             ]
         );
@@ -187,24 +140,22 @@ class PocketImport extends AbstractImport
 
         if ($this->producer) {
             $this->parseEntriesForProducer($entries['list']);
-
-            return true;
+        } else {
+            $this->parseEntries($entries['list']);
         }
 
-        $this->parseEntries($entries['list']);
+        // if we retrieve exactly the amount of items requested it means we can get more
+        // re-call import and offset item by the amount previous received:
+        //  - first call get 5k offset 0
+        //  - second call get 5k offset 5k
+        //  - and so on
+        if (self::NB_ELEMENTS === count($entries['list'])) {
+            ++$run;
 
-        return true;
-    }
+            return $this->import(self::NB_ELEMENTS * $run);
+        }
 
-    /**
-     * {@inheritdoc}
-     */
-    public function getSummary()
-    {
-        return [
-            'skipped' => $this->skippedEntries,
-            'imported' => $this->importedEntries,
-        ];
+        return true;
     }
 
     /**
@@ -218,36 +169,13 @@ class PocketImport extends AbstractImport
     }
 
     /**
-     * @see https://getpocket.com/developer/docs/v3/retrieve
+     * {@inheritdoc}
      *
-     * @param array $entries
+     * @see https://getpocket.com/developer/docs/v3/retrieve
      */
-    private function parseEntries(array $entries)
-    {
-        $i = 1;
-
-        foreach ($entries as $pocketEntry) {
-            $entry = $this->parseEntry($pocketEntry);
-
-            if (null === $entry) {
-                continue;
-            }
-
-            // flush every 20 entries
-            if (($i % 20) === 0) {
-                $this->em->flush();
-                $this->em->clear($entry);
-            }
-
-            ++$i;
-        }
-
-        $this->em->flush();
-    }
-
-    public function parseEntry(array $pocketEntry)
+    public function parseEntry(array $importedEntry)
     {
-        $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
+        $url = isset($importedEntry['resolved_url']) && '' !== $importedEntry['resolved_url'] ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
 
         $existingEntry = $this->em
             ->getRepository('WallabagCoreBundle:Entry')
@@ -260,47 +188,43 @@ class PocketImport extends AbstractImport
         }
 
         $entry = new Entry($this->user);
-        $entry = $this->fetchContent($entry, $url);
-
-        // jump to next entry in case of problem while getting content
-        if (false === $entry) {
-            ++$this->skippedEntries;
+        $entry->setUrl($url);
 
-            return;
-        }
+        // update entry with content (in case fetching failed, the given entry will be return)
+        $this->fetchContent($entry, $url);
 
         // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
-        if ($pocketEntry['status'] == 1 || $this->markAsRead) {
-            $entry->setArchived(true);
-        }
+        $entry->setArchived(1 === $importedEntry['status'] || $this->markAsRead);
 
         // 0 or 1 - 1 If the item is starred
-        if ($pocketEntry['favorite'] == 1) {
-            $entry->setStarred(true);
-        }
+        $entry->setStarred(1 === $importedEntry['favorite']);
 
         $title = 'Untitled';
-        if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
-            $title = $pocketEntry['resolved_title'];
-        } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
-            $title = $pocketEntry['given_title'];
+        if (isset($importedEntry['resolved_title']) && '' !== $importedEntry['resolved_title']) {
+            $title = $importedEntry['resolved_title'];
+        } elseif (isset($importedEntry['given_title']) && '' !== $importedEntry['given_title']) {
+            $title = $importedEntry['given_title'];
         }
 
         $entry->setTitle($title);
-        $entry->setUrl($url);
 
         // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
-        if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
-            $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
+        if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
+            $entry->setPreviewPicture($importedEntry['images'][1]['src']);
         }
 
-        if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
-            $this->contentProxy->assignTagsToEntry(
+        if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
+            $this->tagsAssigner->assignTagsToEntry(
                 $entry,
-                array_keys($pocketEntry['tags'])
+                array_keys($importedEntry['tags']),
+                $this->em->getUnitOfWork()->getScheduledEntityInsertions()
             );
         }
 
+        if (!empty($importedEntry['time_added'])) {
+            $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
+        }
+
         $this->em->persist($entry);
         ++$this->importedEntries;
 
@@ -308,24 +232,12 @@ class PocketImport extends AbstractImport
     }
 
     /**
-     * Faster parse entries for Producer.
-     * We don't care to make check at this time. They'll be done by the consumer.
-     *
-     * @param array $entries
+     * {@inheritdoc}
      */
-    public function parseEntriesForProducer($entries)
+    protected function setEntryAsRead(array $importedEntry)
     {
-        foreach ($entries as $pocketEntry) {
-            // set userId for the producer (it won't know which user is connected)
-            $pocketEntry['userId'] = $this->user->getId();
+        $importedEntry['status'] = '1';
 
-            if ($this->markAsRead) {
-                $pocketEntry['status'] = 1;
-            }
-
-            ++$this->importedEntries;
-
-            $this->producer->publish(json_encode($pocketEntry));
-        }
+        return $importedEntry;
     }
 }