X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=src%2FWallabag%2FImportBundle%2FImport%2FPocketImport.php;h=7d38826b2ac7e935ab47450119fa9fdcafbcf1d0;hb=f808b01692a835673f328d7221ba8c212caa9b61;hp=e5c86f07b5d47d9b86258bdba4185bac003dbdfc;hpb=7ec2897ee0ad190dcb9f77032d785f2f9661b754;p=github%2Fwallabag%2Fwallabag.git diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index e5c86f07..7d38826b 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,29 +2,24 @@ namespace Wallabag\ImportBundle\Import; -use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; -use Symfony\Component\HttpFoundation\Session\Session; -use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; +use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\CoreBundle\Entity\Tag; -use Wallabag\CoreBundle\Tools\Utils; -class PocketImport implements ImportInterface +class PocketImport extends AbstractImport { - private $user; - private $session; - private $em; - private $consumerKey; - private $skippedEntries = 0; - private $importedEntries = 0; - - public function __construct(TokenStorageInterface $tokenStorage, Session $session, EntityManager $em, $consumerKey) + const NB_ELEMENTS = 5000; + private $client; + private $accessToken; + + /** + * Only used for test purpose. + * + * @return string + */ + public function getAccessToken() { - $this->user = $tokenStorage->getToken()->getUser(); - $this->session = $session; - $this->em = $em; - $this->consumerKey = $consumerKey; + return $this->accessToken; } /** @@ -38,79 +33,129 @@ class PocketImport implements ImportInterface /** * {@inheritdoc} */ - public function getDescription() + public function getUrl() { - return 'This importer will import all your Pocket data.'; + return 'import_pocket'; } /** * {@inheritdoc} */ - public function oAuthRequest($redirectUri, $callbackUri) + public function getDescription() + { + return 'import.pocket.description'; + } + + /** + * Return the oauth url to authenticate the client. + * + * @param string $redirectUri Redirect url in case of error + * + * @return string|false request_token for callback method + */ + public function getRequestToken($redirectUri) { $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'redirect_uri' => $redirectUri, ]), ] ); - $response = $this->client->send($request); - $values = $response->json(); + try { + $response = $this->client->send($request); + } catch (RequestException $e) { + $this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]); - // store code in session for callback method - $this->session->set('pocketCode', $values['code']); + return false; + } - return 'https://getpocket.com/auth/authorize?request_token='.$values['code'].'&redirect_uri='.$callbackUri; + return $response->json()['code']; } /** - * {@inheritdoc} + * Usually called by the previous callback to authorize the client. + * Then it return a token that can be used for next requests. + * + * @param string $code request_token from getRequestToken + * + * @return bool */ - public function oAuthAuthorize() + public function authorize($code) { $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, - 'code' => $this->session->get('pocketCode'), + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), + 'code' => $code, ]), ] ); - $response = $this->client->send($request); + try { + $response = $this->client->send($request); + } catch (RequestException $e) { + $this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]); + + return false; + } + + $this->accessToken = $response->json()['access_token']; - return $response->json()['access_token']; + return true; } /** * {@inheritdoc} */ - public function import($accessToken) + public function import($offset = 0) { + static $run = 0; + $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, - 'access_token' => $accessToken, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), + 'access_token' => $this->accessToken, 'detailType' => 'complete', 'state' => 'all', - 'sort' => 'oldest', + 'sort' => 'newest', + 'count' => self::NB_ELEMENTS, + 'offset' => $offset, ]), ] ); - $response = $this->client->send($request); + try { + $response = $this->client->send($request); + } catch (RequestException $e) { + $this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]); + + return false; + } + $entries = $response->json(); - $this->parsePocketEntries($entries['list']); + if ($this->producer) { + $this->parseEntriesForProducer($entries['list']); + } else { + $this->parseEntries($entries['list']); + } - $this->session->getFlashBag()->add( - 'notice', - $this->importedEntries.' entries imported, '.$this->skippedEntries.' already saved.' - ); + // if we retrieve exactly the amount of items requested it means we can get more + // re-call import and offset item by the amount previous received: + // - first call get 5k offset 0 + // - second call get 5k offset 5k + // - and so on + if (count($entries['list']) === self::NB_ELEMENTS) { + ++$run; + + return $this->import(self::NB_ELEMENTS * $run); + } + + return true; } /** @@ -124,108 +169,75 @@ class PocketImport implements ImportInterface } /** - * Returns the good title for current entry. - * - * @param $pocketEntry + * {@inheritdoc} * - * @return string + * @see https://getpocket.com/developer/docs/v3/retrieve */ - private function guessTitle($pocketEntry) + public function parseEntry(array $importedEntry) { - if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { - return $pocketEntry['resolved_title']; - } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { - return $pocketEntry['given_title']; - } + $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] !== '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; - return 'Untitled'; - } + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->user->getId()); - /** - * Returns the good URL for current entry. - * - * @param $pocketEntry - * - * @return string - */ - private function guessURL($pocketEntry) - { - if (isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '') { - return $pocketEntry['resolved_url']; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - return $pocketEntry['given_url']; - } - - private function assignTagsToEntry(Entry $entry, $tags) - { - foreach ($tags as $tag) { - $label = trim($tag['tag']); - $tagEntity = $this->em - ->getRepository('WallabagCoreBundle:Tag') - ->findOneByLabelAndUserId($label, $this->user->getId()); - - if (is_object($tagEntity)) { - $entry->addTag($tagEntity); - } else { - $newTag = new Tag($this->user); - $newTag->setLabel($label); - $entry->addTag($newTag); - } - $this->em->flush(); + return; } - } - /** - * @param $entries - */ - private function parsePocketEntries($entries) - { - foreach ($entries as $pocketEntry) { - $entry = new Entry($this->user); - $url = $this->guessURL($pocketEntry); + $entry = new Entry($this->user); + $entry->setUrl($url); - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->existByUrlAndUserId($url, $this->user->getId()); + // update entry with content (in case fetching failed, the given entry will be return) + $this->fetchContent($entry, $url); - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } + // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted + $entry->setArchived($importedEntry['status'] === 1 || $this->markAsRead); - $entry->setUrl($url); - $entry->setDomainName(parse_url($url, PHP_URL_HOST)); + // 0 or 1 - 1 If the item is starred + $entry->setStarred($importedEntry['favorite'] === 1); - if ($pocketEntry['status'] == 1) { - $entry->setArchived(true); - } - if ($pocketEntry['favorite'] == 1) { - $entry->setStarred(true); - } + $title = 'Untitled'; + if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] !== '') { + $title = $importedEntry['resolved_title']; + } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] !== '') { + $title = $importedEntry['given_title']; + } - $entry->setTitle($this->guessTitle($pocketEntry)); + $entry->setTitle($title); - if (isset($pocketEntry['excerpt'])) { - $entry->setContent($pocketEntry['excerpt']); - } + // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image + if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { + $entry->setPreviewPicture($importedEntry['images'][1]['src']); + } - if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0) { - $entry->setPreviewPicture($pocketEntry['image']['src']); - } + if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { + $this->tagsAssigner->assignTagsToEntry( + $entry, + array_keys($importedEntry['tags']), + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } - if (isset($pocketEntry['word_count'])) { - $entry->setReadingTime(Utils::convertWordsToMinutes($pocketEntry['word_count'])); - } + if (!empty($importedEntry['time_added'])) { + $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added'])); + } - if (!empty($pocketEntry['tags'])) { - $this->assignTagsToEntry($entry, $pocketEntry['tags']); - } + $this->em->persist($entry); + ++$this->importedEntries; - $this->em->persist($entry); - ++$this->importedEntries; - } + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['status'] = '1'; - $this->em->flush(); + return $importedEntry; } }