X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=src%2FWallabag%2FImportBundle%2FImport%2FPocketImport.php;h=d364338996a652c83684d4d44ce39d960d77364d;hb=9f8f188d928b47503d39348c5990379a572b570a;hp=4499ce6993563f41d1eccfddbf1b22f1743cb9e7;hpb=0d42217e4e8210dd2cf86f35ba9662ca02c8a2dc;p=github%2Fwallabag%2Fwallabag.git diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 4499ce69..d3643389 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,41 +2,24 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; -use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\CoreBundle\Helper\ContentProxy; -use Craue\ConfigBundle\Util\Config; -class PocketImport implements ImportInterface +class PocketImport extends AbstractImport { - private $user; - private $em; - private $contentProxy; - private $logger; + const NB_ELEMENTS = 5000; private $client; - private $consumerKey; - private $skippedEntries = 0; - private $importedEntries = 0; - private $markAsRead; - protected $accessToken; + private $accessToken; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) - { - $this->user = $tokenStorage->getToken()->getUser(); - $this->em = $em; - $this->contentProxy = $contentProxy; - $this->consumerKey = $craueConfig->get('pocket_consumer_key'); - $this->logger = new NullLogger(); - } - - public function setLogger(LoggerInterface $logger) + /** + * Only used for test purpose. + * + * @return string + */ + public function getAccessToken() { - $this->logger = $logger; + return $this->accessToken; } /** @@ -68,14 +51,14 @@ class PocketImport implements ImportInterface * * @param string $redirectUri Redirect url in case of error * - * @return string request_token for callback method + * @return string|false request_token for callback method */ public function getRequestToken($redirectUri) { $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'redirect_uri' => $redirectUri, ]), ] @@ -105,7 +88,7 @@ class PocketImport implements ImportInterface $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'code' => $code, ]), ] @@ -124,39 +107,23 @@ class PocketImport implements ImportInterface return true; } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; - - return $this; - } - - /** - * Get whether articles must be all marked as read. - */ - public function getMarkAsRead() - { - return $this->markAsRead; - } - /** * {@inheritdoc} */ - public function import() + public function import($offset = 0) { + static $run = 0; + $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'access_token' => $this->accessToken, 'detailType' => 'complete', 'state' => 'all', - 'sort' => 'oldest', + 'sort' => 'newest', + 'count' => self::NB_ELEMENTS, + 'offset' => $offset, ]), ] ); @@ -171,20 +138,24 @@ class PocketImport implements ImportInterface $entries = $response->json(); - $this->parseEntries($entries['list']); + if ($this->producer) { + $this->parseEntriesForProducer($entries['list']); + } else { + $this->parseEntries($entries['list']); + } - return true; - } + // if we retrieve exactly the amount of items requested it means we can get more + // re-call import and offset item by the amount previous received: + // - first call get 5k offset 0 + // - second call get 5k offset 5k + // - and so on + if (self::NB_ELEMENTS === \count($entries['list'])) { + ++$run; - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; + return $this->import(self::NB_ELEMENTS * $run); + } + + return true; } /** @@ -198,70 +169,87 @@ class PocketImport implements ImportInterface } /** - * @see https://getpocket.com/developer/docs/v3/retrieve + * {@inheritdoc} + */ + public function validateEntry(array $importedEntry) + { + if (empty($importedEntry['resolved_url']) && empty($importedEntry['given_url'])) { + return false; + } + + return true; + } + + /** + * {@inheritdoc} * - * @param $entries + * @see https://getpocket.com/developer/docs/v3/retrieve */ - private function parseEntries($entries) + public function parseEntry(array $importedEntry) { - $i = 1; - - foreach ($entries as $pocketEntry) { - $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($url, $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } - - $entry = new Entry($this->user); - $entry = $this->contentProxy->updateEntry($entry, $url); - - // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($pocketEntry['status'] == 1 || $this->markAsRead) { - $entry->setArchived(true); - } - - // 0 or 1 - 1 If the item is favorited - if ($pocketEntry['favorite'] == 1) { - $entry->setStarred(true); - } - - $title = 'Untitled'; - if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { - $title = $pocketEntry['resolved_title']; - } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { - $title = $pocketEntry['given_title']; - } - - $entry->setTitle($title); - - // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image - if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { - $entry->setPreviewPicture($pocketEntry['images'][1]['src']); - } - - if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { - $this->contentProxy->assignTagsToEntry( - $entry, - array_keys($pocketEntry['tags']) - ); - } - - $this->em->persist($entry); - ++$this->importedEntries; - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } - ++$i; + $url = isset($importedEntry['resolved_url']) && '' !== $importedEntry['resolved_url'] ? $importedEntry['resolved_url'] : $importedEntry['given_url']; + + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; } - $this->em->flush(); + $entry = new Entry($this->user); + $entry->setUrl($url); + + // update entry with content (in case fetching failed, the given entry will be return) + $this->fetchContent($entry, $url); + + // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted + $entry->setArchived(1 === $importedEntry['status'] || $this->markAsRead); + + // 0 or 1 - 1 If the item is starred + $entry->setStarred(1 === $importedEntry['favorite']); + + $title = 'Untitled'; + if (isset($importedEntry['resolved_title']) && '' !== $importedEntry['resolved_title']) { + $title = $importedEntry['resolved_title']; + } elseif (isset($importedEntry['given_title']) && '' !== $importedEntry['given_title']) { + $title = $importedEntry['given_title']; + } + + $entry->setTitle($title); + + // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image + if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { + $entry->setPreviewPicture($importedEntry['images'][1]['src']); + } + + if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { + $this->tagsAssigner->assignTagsToEntry( + $entry, + array_keys($importedEntry['tags']), + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } + + if (!empty($importedEntry['time_added'])) { + $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added'])); + } + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['status'] = '1'; + + return $importedEntry; } }