namespace Wallabag\ImportBundle\Import;
-use OldSound\RabbitMqBundle\RabbitMq\Producer;
-use Psr\Log\NullLogger;
-use Doctrine\ORM\EntityManager;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
-use Symfony\Component\Security\Core\User\UserInterface;
use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\CoreBundle\Helper\ContentProxy;
-use Craue\ConfigBundle\Util\Config;
class PocketImport extends AbstractImport
{
- private $user;
+ const NB_ELEMENTS = 5000;
private $client;
- private $consumerKey;
- private $skippedEntries = 0;
- private $importedEntries = 0;
- private $markAsRead;
- private $producer;
- protected $accessToken;
-
- public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
- {
- $this->em = $em;
- $this->contentProxy = $contentProxy;
- $this->consumerKey = $craueConfig->get('pocket_consumer_key');
- $this->logger = new NullLogger();
- }
-
- /**
- * Set RabbitMQ Producer to send each entry to a queue.
- * This method should be called when user has enabled RabbitMQ.
- *
- * @param Producer $producer
- */
- public function setRabbitmqProducer(Producer $producer)
- {
- $this->producer = $producer;
- }
+ private $accessToken;
/**
- * Set current user.
- * Could the current *connected* user or one retrieve by the consumer.
+ * Only used for test purpose.
*
- * @param UserInterface $user
+ * @return string
*/
- public function setUser(UserInterface $user)
+ public function getAccessToken()
{
- $this->user = $user;
+ return $this->accessToken;
}
/**
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
[
'body' => json_encode([
- 'consumer_key' => $this->consumerKey,
+ 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'redirect_uri' => $redirectUri,
]),
]
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
[
'body' => json_encode([
- 'consumer_key' => $this->consumerKey,
+ 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'code' => $code,
]),
]
return true;
}
- /**
- * Set whether articles must be all marked as read.
- *
- * @param bool $markAsRead
- */
- public function setMarkAsRead($markAsRead)
- {
- $this->markAsRead = $markAsRead;
-
- return $this;
- }
-
- /**
- * Get whether articles must be all marked as read.
- */
- public function getMarkAsRead()
- {
- return $this->markAsRead;
- }
-
/**
* {@inheritdoc}
*/
- public function import()
+ public function import($offset = 0)
{
+ static $run = 0;
+
$request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
[
'body' => json_encode([
- 'consumer_key' => $this->consumerKey,
+ 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
'access_token' => $this->accessToken,
'detailType' => 'complete',
'state' => 'all',
- 'sort' => 'oldest',
+ 'sort' => 'newest',
+ 'count' => self::NB_ELEMENTS,
+ 'offset' => $offset,
]),
]
);
if ($this->producer) {
$this->parseEntriesForProducer($entries['list']);
-
- return true;
+ } else {
+ $this->parseEntries($entries['list']);
}
- $this->parseEntries($entries['list']);
+ // if we retrieve exactly the amount of items requested it means we can get more
+ // re-call import and offset item by the amount previous received:
+ // - first call get 5k offset 0
+ // - second call get 5k offset 5k
+ // - and so on
+ if (self::NB_ELEMENTS === \count($entries['list'])) {
+ ++$run;
- return true;
- }
+ return $this->import(self::NB_ELEMENTS * $run);
+ }
- /**
- * {@inheritdoc}
- */
- public function getSummary()
- {
- return [
- 'skipped' => $this->skippedEntries,
- 'imported' => $this->importedEntries,
- ];
+ return true;
}
/**
}
/**
- * @see https://getpocket.com/developer/docs/v3/retrieve
- *
- * @param array $entries
+ * {@inheritdoc}
*/
- private function parseEntries(array $entries)
+ public function validateEntry(array $importedEntry)
{
- $i = 1;
-
- foreach ($entries as $pocketEntry) {
- $entry = $this->parseEntry($pocketEntry);
-
- if (null === $entry) {
- continue;
- }
-
- // flush every 20 entries
- if (($i % 20) === 0) {
- $this->em->flush();
- $this->em->clear($entry);
- }
-
- ++$i;
+ if (empty($importedEntry['resolved_url']) && empty($importedEntry['given_url'])) {
+ return false;
}
- $this->em->flush();
+ return true;
}
- public function parseEntry(array $pocketEntry)
+ /**
+ * {@inheritdoc}
+ *
+ * @see https://getpocket.com/developer/docs/v3/retrieve
+ */
+ public function parseEntry(array $importedEntry)
{
- $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
+ $url = isset($importedEntry['resolved_url']) && '' !== $importedEntry['resolved_url'] ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
$existingEntry = $this->em
->getRepository('WallabagCoreBundle:Entry')
}
$entry = new Entry($this->user);
- $entry = $this->fetchContent($entry, $url);
-
- // jump to next entry in case of problem while getting content
- if (false === $entry) {
- ++$this->skippedEntries;
+ $entry->setUrl($url);
- return;
- }
+ // update entry with content (in case fetching failed, the given entry will be return)
+ $this->fetchContent($entry, $url);
// 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
- if ($pocketEntry['status'] == 1 || $this->markAsRead) {
- $entry->setArchived(true);
- }
+ $entry->updateArchived(1 === $importedEntry['status'] || $this->markAsRead);
// 0 or 1 - 1 If the item is starred
- if ($pocketEntry['favorite'] == 1) {
- $entry->setStarred(true);
- }
+ $entry->setStarred(1 === $importedEntry['favorite']);
$title = 'Untitled';
- if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
- $title = $pocketEntry['resolved_title'];
- } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
- $title = $pocketEntry['given_title'];
+ if (isset($importedEntry['resolved_title']) && '' !== $importedEntry['resolved_title']) {
+ $title = $importedEntry['resolved_title'];
+ } elseif (isset($importedEntry['given_title']) && '' !== $importedEntry['given_title']) {
+ $title = $importedEntry['given_title'];
}
$entry->setTitle($title);
- $entry->setUrl($url);
// 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
- if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
- $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
+ if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
+ $entry->setPreviewPicture($importedEntry['images'][1]['src']);
}
- if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
- $this->contentProxy->assignTagsToEntry(
+ if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
+ $this->tagsAssigner->assignTagsToEntry(
$entry,
- array_keys($pocketEntry['tags'])
+ array_keys($importedEntry['tags']),
+ $this->em->getUnitOfWork()->getScheduledEntityInsertions()
);
}
+ if (!empty($importedEntry['time_added'])) {
+ $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
+ }
+
$this->em->persist($entry);
++$this->importedEntries;
}
/**
- * Faster parse entries for Producer.
- * We don't care to make check at this time. They'll be done by the consumer.
- *
- * @param array $entries
+ * {@inheritdoc}
*/
- public function parseEntriesForProducer($entries)
+ protected function setEntryAsRead(array $importedEntry)
{
- foreach ($entries as $pocketEntry) {
- // set userId for the producer (it won't know which user is connected)
- $pocketEntry['userId'] = $this->user->getId();
+ $importedEntry['status'] = '1';
- if ($this->markAsRead) {
- $pocketEntry['status'] = 1;
- }
-
- ++$this->importedEntries;
-
- $this->producer->publish(json_encode($pocketEntry));
- }
+ return $importedEntry;
}
}
projects / github / wallabag / wallabag.git / blobdiff