namespace Wallabag\ImportBundle\Import;
-use Psr\Log\LoggerInterface;
-use Psr\Log\NullLogger;
-use Doctrine\ORM\EntityManager;
-use GuzzleHttp\Client;
-use GuzzleHttp\Exception\RequestException;
-use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
+use Http\Client\Common\HttpMethodsClient;
+use Http\Client\Common\Plugin\ErrorPlugin;
+use Http\Client\Common\PluginClient;
+use Http\Client\Exception\RequestException;
+use Http\Client\HttpClient;
+use Http\Discovery\MessageFactoryDiscovery;
+use Http\Message\MessageFactory;
+use Psr\Http\Message\ResponseInterface;
use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\CoreBundle\Entity\Tag;
-use Wallabag\CoreBundle\Helper\ContentProxy;
-class PocketImport implements ImportInterface
+class PocketImport extends AbstractImport
{
- private $user;
- private $em;
- private $contentProxy;
- private $logger;
+ const NB_ELEMENTS = 5000;
+ /**
+ * @var HttpMethodsClient
+ */
private $client;
- private $consumerKey;
- private $skippedEntries = 0;
- private $importedEntries = 0;
- protected $accessToken;
- private $translator;
+ private $accessToken;
- public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, $consumerKey)
- {
- $this->user = $tokenStorage->getToken()->getUser();
- $this->em = $em;
- $this->contentProxy = $contentProxy;
- $this->consumerKey = $consumerKey;
- $this->logger = new NullLogger();
- }
-
- public function setLogger(LoggerInterface $logger)
+ /**
+ * Only used for test purpose.
+ *
+ * @return string
+ */
+ public function getAccessToken()
{
- $this->logger = $logger;
+ return $this->accessToken;
}
/**
*/
public function getDescription()
{
- return 'This importer will import all your Pocket data. Pocket doesn\'t allow us to retrieve content from their service, so the readable content of each article will be re-fetched by wallabag.';
+ return 'import.pocket.description';
}
/**
*
* @param string $redirectUri Redirect url in case of error
*
- * @return string request_token for callback method
+ * @return string|false request_token for callback method
*/
public function getRequestToken($redirectUri)
{
- $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
- [
- 'body' => json_encode([
- 'consumer_key' => $this->consumerKey,
- 'redirect_uri' => $redirectUri,
- ]),
- ]
- );
-
try {
- $response = $this->client->send($request);
+ $response = $this->client->post('https://getpocket.com/v3/oauth/request', [], json_encode([
+ 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
+ 'redirect_uri' => $redirectUri,
+ ]));
} catch (RequestException $e) {
$this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]);
return false;
}
- return $response->json()['code'];
+ return $this->jsonDecode($response)['code'];
}
/**
*/
public function authorize($code)
{
- $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
- [
- 'body' => json_encode([
- 'consumer_key' => $this->consumerKey,
- 'code' => $code,
- ]),
- ]
- );
-
try {
- $response = $this->client->send($request);
+ $response = $this->client->post('https://getpocket.com/v3/oauth/authorize', [], json_encode([
+ 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
+ 'code' => $code,
+ ]));
} catch (RequestException $e) {
$this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]);
return false;
}
- $this->accessToken = $response->json()['access_token'];
+ $this->accessToken = $this->jsonDecode($response)['access_token'];
return true;
}
/**
* {@inheritdoc}
*/
- public function import()
+ public function import($offset = 0)
{
- $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
- [
- 'body' => json_encode([
- 'consumer_key' => $this->consumerKey,
- 'access_token' => $this->accessToken,
- 'detailType' => 'complete',
- 'state' => 'all',
- 'sort' => 'oldest',
- ]),
- ]
- );
+ static $run = 0;
try {
- $response = $this->client->send($request);
+ $response = $this->client->post('https://getpocket.com/v3/get', [], json_encode([
+ 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
+ 'access_token' => $this->accessToken,
+ 'detailType' => 'complete',
+ 'state' => 'all',
+ 'sort' => 'newest',
+ 'count' => self::NB_ELEMENTS,
+ 'offset' => $offset,
+ ]));
} catch (RequestException $e) {
$this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]);
return false;
}
- $entries = $response->json();
+ $entries = $this->jsonDecode($response);
- $this->parseEntries($entries['list']);
+ if ($this->producer) {
+ $this->parseEntriesForProducer($entries['list']);
+ } else {
+ $this->parseEntries($entries['list']);
+ }
+
+ // if we retrieve exactly the amount of items requested it means we can get more
+ // re-call import and offset item by the amount previous received:
+ // - first call get 5k offset 0
+ // - second call get 5k offset 5k
+ // - and so on
+ if (self::NB_ELEMENTS === \count($entries['list'])) {
+ ++$run;
+
+ return $this->import(self::NB_ELEMENTS * $run);
+ }
return true;
}
/**
- * {@inheritdoc}
+ * Set the Http client.
*/
- public function getSummary()
+ public function setClient(HttpClient $client, MessageFactory $messageFactory = null)
{
- return [
- 'skipped' => $this->skippedEntries,
- 'imported' => $this->importedEntries,
- ];
+ $this->client = new HttpMethodsClient(new PluginClient($client, [new ErrorPlugin()]), $messageFactory ?: MessageFactoryDiscovery::find());
}
/**
- * Set the Guzzle client.
- *
- * @param Client $client
+ * {@inheritdoc}
*/
- public function setClient(Client $client)
+ public function validateEntry(array $importedEntry)
{
- $this->client = $client;
+ if (empty($importedEntry['resolved_url']) && empty($importedEntry['given_url'])) {
+ return false;
+ }
+
+ return true;
}
/**
- * @todo move that in a more global place
+ * {@inheritdoc}
+ *
+ * @see https://getpocket.com/developer/docs/v3/retrieve
*/
- private function assignTagsToEntry(Entry $entry, $tags)
+ public function parseEntry(array $importedEntry)
{
- foreach ($tags as $tag) {
- $label = trim($tag['tag']);
- $tagEntity = $this->em
- ->getRepository('WallabagCoreBundle:Tag')
- ->findOneByLabel($label);
-
- if (is_object($tagEntity)) {
- $entry->addTag($tagEntity);
- } else {
- $newTag = new Tag();
- $newTag->setLabel($label);
-
- $entry->addTag($newTag);
- }
- $this->em->flush();
+ $url = isset($importedEntry['resolved_url']) && '' !== $importedEntry['resolved_url'] ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
+
+ $existingEntry = $this->em
+ ->getRepository('WallabagCoreBundle:Entry')
+ ->findByUrlAndUserId($url, $this->user->getId());
+
+ if (false !== $existingEntry) {
+ ++$this->skippedEntries;
+
+ return;
+ }
+
+ $entry = new Entry($this->user);
+ $entry->setUrl($url);
+
+ // update entry with content (in case fetching failed, the given entry will be return)
+ $this->fetchContent($entry, $url);
+
+ // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
+ $entry->updateArchived(1 === (int) $importedEntry['status'] || $this->markAsRead);
+
+ // 0 or 1 - 1 if the item is starred
+ $entry->setStarred(1 === (int) $importedEntry['favorite']);
+
+ $title = 'Untitled';
+ if (isset($importedEntry['resolved_title']) && '' !== $importedEntry['resolved_title']) {
+ $title = $importedEntry['resolved_title'];
+ } elseif (isset($importedEntry['given_title']) && '' !== $importedEntry['given_title']) {
+ $title = $importedEntry['given_title'];
+ }
+
+ $entry->setTitle($title);
+
+ // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
+ if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
+ $entry->setPreviewPicture($importedEntry['images'][1]['src']);
+ }
+
+ if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
+ $this->tagsAssigner->assignTagsToEntry(
+ $entry,
+ array_keys($importedEntry['tags']),
+ $this->em->getUnitOfWork()->getScheduledEntityInsertions()
+ );
}
+
+ if (!empty($importedEntry['time_added'])) {
+ $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
+ }
+
+ $this->em->persist($entry);
+ ++$this->importedEntries;
+
+ return $entry;
}
/**
- * @see https://getpocket.com/developer/docs/v3/retrieve
- *
- * @param $entries
+ * {@inheritdoc}
*/
- private function parseEntries($entries)
+ protected function setEntryAsRead(array $importedEntry)
+ {
+ $importedEntry['status'] = '1';
+
+ return $importedEntry;
+ }
+
+ protected function jsonDecode(ResponseInterface $response)
{
- $i = 1;
-
- foreach ($entries as $pocketEntry) {
- $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url'];
-
- $existingEntry = $this->em
- ->getRepository('WallabagCoreBundle:Entry')
- ->findByUrlAndUserId($url, $this->user->getId());
-
- if (false !== $existingEntry) {
- ++$this->skippedEntries;
- continue;
- }
-
- $entry = new Entry($this->user);
- $entry = $this->contentProxy->updateEntry($entry, $url);
-
- // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
- if ($pocketEntry['status'] == 1) {
- $entry->setArchived(true);
- }
-
- // 0 or 1 - 1 If the item is favorited
- if ($pocketEntry['favorite'] == 1) {
- $entry->setStarred(true);
- }
-
- $title = 'Untitled';
- if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') {
- $title = $pocketEntry['resolved_title'];
- } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') {
- $title = $pocketEntry['given_title'];
- }
-
- $entry->setTitle($title);
-
- // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
- if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
- $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
- }
-
- if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
- $this->assignTagsToEntry($entry, $pocketEntry['tags']);
- }
-
- $this->em->persist($entry);
- ++$this->importedEntries;
-
- // flush every 20 entries
- if (($i % 20) === 0) {
- $this->em->flush();
- }
- ++$i;
+ $data = json_decode((string) $response->getBody(), true);
+
+ if (JSON_ERROR_NONE !== json_last_error()) {
+ throw new \InvalidArgumentException('Unable to parse JSON data: ' . json_last_error_msg());
}
- $this->em->flush();
+ return $data;
}
}