From 03e3753f6bd36f12c0757c76b49b683c49de48ae Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Thu, 1 Sep 2016 08:00:30 +0200 Subject: Add Readability import Based on the JSON export instead of the API (which will be shutting down by the September 30, 2016) --- .../ImportBundle/Import/ReadabilityImport.php | 181 +++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 src/Wallabag/ImportBundle/Import/ReadabilityImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php new file mode 100644 index 00000000..abea81a7 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -0,0 +1,181 @@ +user = $user; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'Readability'; + } + + /** + * {@inheritdoc} + */ + public function getUrl() + { + return 'import_readability'; + } + + /** + * {@inheritdoc} + */ + public function getDescription() + { + return 'import.readability.description'; + } + + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Get whether articles must be all marked as read. + */ + public function getMarkAsRead() + { + return $this->markAsRead; + } + + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + ]; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('ReadabilityImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data) || empty($data['bookmarks'])) { + return false; + } + + $this->parseEntries($data['bookmarks']); + + return true; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) + { + $i = 1; + + foreach ($entries as $importedEntry) { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + continue; + } + + $data = [ + 'title' => $importedEntry['article__title'], + // 'html' => $importedEntry['article__excerpt'], + 'url' => $importedEntry['article__url'], + 'content_type' => '', + 'language' => '', + 'is_archived' => $importedEntry['archive'] || $this->markAsRead, + 'is_starred' => $importedEntry['favorite'], + ]; + + $entry = $this->fetchContent( + new Entry($this->user), + $data['url'], + $data + ); + + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; + continue; + } + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$i; + } + + $this->em->flush(); + } +} -- cgit v1.2.3 From a1a107705948c724aca7326f8550c336a25a6364 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Thu, 1 Sep 2016 09:57:02 +0200 Subject: Add tests on ReadabilityImport --- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index abea81a7..37b160c5 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -8,7 +8,6 @@ use Wallabag\UserBundle\Entity\User; class ReadabilityImport extends AbstractImport { private $user; - private $client; private $skippedEntries = 0; private $importedEntries = 0; private $filepath; @@ -143,7 +142,6 @@ class ReadabilityImport extends AbstractImport $data = [ 'title' => $importedEntry['article__title'], - // 'html' => $importedEntry['article__excerpt'], 'url' => $importedEntry['article__url'], 'content_type' => '', 'language' => '', -- cgit v1.2.3 From 58fadbc9df3f8b735c04995919b6cf913ca6a977 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 15:53:16 +0200 Subject: Fix error on EntityManager clear Introduced in the recent 2.5.5 release. Also updated deps. --- src/Wallabag/ImportBundle/Import/PocketImport.php | 5 ++++- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 5 ++++- src/Wallabag/ImportBundle/Import/WallabagImport.php | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 798cfdae..841f829d 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -255,7 +255,10 @@ class PocketImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } ++$i; } diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 37b160c5..7bae647d 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -169,7 +169,10 @@ class ReadabilityImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } ++$i; } diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index a1cc085b..9cd3dcb8 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -172,7 +172,10 @@ abstract class WallabagImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } ++$i; } -- cgit v1.2.3 From 88e88016b9a86d76940082b62c85c3756f461cb2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 16:30:01 +0200 Subject: Clearing entities in the loop fail on Postgres It looks like when you clear entities on Postgres some references are lost and tags are not saved :-/ --- src/Wallabag/ImportBundle/Import/PocketImport.php | 5 +---- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 5 +---- src/Wallabag/ImportBundle/Import/WallabagImport.php | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 841f829d..a6f905b1 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -255,14 +255,11 @@ class PocketImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); } ++$i; } $this->em->flush(); + $this->em->clear(); } } diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 7bae647d..c7cfe15d 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -169,14 +169,11 @@ class ReadabilityImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); } ++$i; } $this->em->flush(); + $this->em->clear(); } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 9cd3dcb8..581ec178 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -172,15 +172,12 @@ abstract class WallabagImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); } ++$i; } $this->em->flush(); + $this->em->clear(); } /** -- cgit v1.2.3 From 56c778b4152a1b886353933276ee3626e4e8c004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 15 Jan 2016 08:24:32 +0100 Subject: 1st draft for rabbitMQ --- src/Wallabag/ImportBundle/Import/PocketImport.php | 35 +++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index a6f905b1..b02894f0 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,6 +2,8 @@ namespace Wallabag\ImportBundle\Import; +use OldSound\RabbitMqBundle\RabbitMq\Producer; +use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; @@ -20,14 +22,18 @@ class PocketImport extends AbstractImport private $importedEntries = 0; private $markAsRead; protected $accessToken; + private $producer; + private $rabbitMQ; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) + public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, $rabbitMQ, Producer $producer) { $this->user = $tokenStorage->getToken()->getUser(); $this->em = $em; $this->contentProxy = $contentProxy; $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); + $this->rabbitMQ = $rabbitMQ; + $this->producer = $producer; } /** @@ -197,7 +203,7 @@ class PocketImport extends AbstractImport { $i = 1; - foreach ($entries as $pocketEntry) { + foreach ($entries as &$pocketEntry) { $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; $existingEntry = $this->em @@ -210,12 +216,15 @@ class PocketImport extends AbstractImport } $entry = new Entry($this->user); - $entry = $this->fetchContent($entry, $url); - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; + if (!$this->rabbitMQ) { + $entry = $this->fetchContent($entry, $url); + + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; + continue; + } } // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted @@ -236,6 +245,7 @@ class PocketImport extends AbstractImport } $entry->setTitle($title); + $entry->setUrl($url); // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { @@ -249,6 +259,9 @@ class PocketImport extends AbstractImport ); } + $pocketEntry['url'] = $url; + $pocketEntry['userId'] = $this->user->getId(); + $this->em->persist($entry); ++$this->importedEntries; @@ -256,10 +269,16 @@ class PocketImport extends AbstractImport if (($i % 20) === 0) { $this->em->flush(); } + ++$i; } $this->em->flush(); - $this->em->clear(); + + if ($this->rabbitMQ) { + foreach ($entries as $entry) { + $this->producer->publish(serialize($entry)); + } + } } } -- cgit v1.2.3 From 40d2a29443df8ef6fdf1f2d09b5ba8808543c245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Mon, 15 Feb 2016 21:30:55 +0100 Subject: Replace RabbitMQ injection with CraueConfiguration --- src/Wallabag/ImportBundle/Import/PocketImport.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index b02894f0..7d1c0c61 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -25,14 +25,14 @@ class PocketImport extends AbstractImport private $producer; private $rabbitMQ; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, $rabbitMQ, Producer $producer) + public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, Producer $producer) { $this->user = $tokenStorage->getToken()->getUser(); $this->em = $em; $this->contentProxy = $contentProxy; $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); - $this->rabbitMQ = $rabbitMQ; + $this->rabbitMQ = $craueConfig->get('rabbitmq'); $this->producer = $producer; } -- cgit v1.2.3 From ef75e1220ebb76a8df019d946460ad612759f0bb Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 3 Sep 2016 17:36:57 +0200 Subject: Send every imported item to the queue Instead of queing real Entry to process, we queue all the item to import from Pocket in a raw format. Then, the worker retrieve that information, find / create the entry and save it. --- src/Wallabag/ImportBundle/Import/PocketImport.php | 177 ++++++++++++++-------- 1 file changed, 112 insertions(+), 65 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 7d1c0c61..27df4917 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -3,12 +3,11 @@ namespace Wallabag\ImportBundle\Import; use OldSound\RabbitMqBundle\RabbitMq\Producer; -use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; -use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; +use Symfony\Component\Security\Core\User\UserInterface; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Helper\ContentProxy; use Craue\ConfigBundle\Util\Config; @@ -21,21 +20,39 @@ class PocketImport extends AbstractImport private $skippedEntries = 0; private $importedEntries = 0; private $markAsRead; - protected $accessToken; private $producer; - private $rabbitMQ; + protected $accessToken; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, Producer $producer) + public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) { - $this->user = $tokenStorage->getToken()->getUser(); $this->em = $em; $this->contentProxy = $contentProxy; $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); - $this->rabbitMQ = $craueConfig->get('rabbitmq'); + } + + /** + * Set RabbitMQ Producer to send each entry to a queue. + * This method should be called when user has enabled RabbitMQ. + * + * @param Producer $producer + */ + public function setRabbitmqProducer(Producer $producer) + { $this->producer = $producer; } + /** + * Set current user. + * Could the current *connected* user or one retrieve by the consumer. + * + * @param UserInterface $user + */ + public function setUser(UserInterface $user) + { + $this->user = $user; + } + /** * {@inheritdoc} */ @@ -168,6 +185,12 @@ class PocketImport extends AbstractImport $entries = $response->json(); + if ($this->producer) { + $this->parseEntriesForProducer($entries['list']); + + return true; + } + $this->parseEntries($entries['list']); return true; @@ -197,88 +220,112 @@ class PocketImport extends AbstractImport /** * @see https://getpocket.com/developer/docs/v3/retrieve * - * @param $entries + * @param array $entries */ - private function parseEntries($entries) + private function parseEntries(array $entries) { $i = 1; - foreach ($entries as &$pocketEntry) { - $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($url, $this->user->getId()); + foreach ($entries as $pocketEntry) { + $entry = $this->parseEntry($pocketEntry); - if (false !== $existingEntry) { - ++$this->skippedEntries; + if (null === $entry) { continue; } - $entry = new Entry($this->user); + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } - if (!$this->rabbitMQ) { - $entry = $this->fetchContent($entry, $url); + ++$i; + } - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; - } - } + $this->em->flush(); + } - // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($pocketEntry['status'] == 1 || $this->markAsRead) { - $entry->setArchived(true); - } + public function parseEntry(array $pocketEntry) + { + $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; - // 0 or 1 - 1 If the item is starred - if ($pocketEntry['favorite'] == 1) { - $entry->setStarred(true); - } + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->user->getId()); - $title = 'Untitled'; - if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { - $title = $pocketEntry['resolved_title']; - } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { - $title = $pocketEntry['given_title']; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - $entry->setTitle($title); - $entry->setUrl($url); + return; + } - // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image - if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { - $entry->setPreviewPicture($pocketEntry['images'][1]['src']); - } + $entry = new Entry($this->user); + $entry = $this->fetchContent($entry, $url); - if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { - $this->contentProxy->assignTagsToEntry( - $entry, - array_keys($pocketEntry['tags']) - ); - } + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; - $pocketEntry['url'] = $url; - $pocketEntry['userId'] = $this->user->getId(); + return; + } - $this->em->persist($entry); - ++$this->importedEntries; + // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted + if ($pocketEntry['status'] == 1 || $this->markAsRead) { + $entry->setArchived(true); + } - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } + // 0 or 1 - 1 If the item is starred + if ($pocketEntry['favorite'] == 1) { + $entry->setStarred(true); + } - ++$i; + $title = 'Untitled'; + if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { + $title = $pocketEntry['resolved_title']; + } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { + $title = $pocketEntry['given_title']; } - $this->em->flush(); + $entry->setTitle($title); + $entry->setUrl($url); + + // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image + if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { + $entry->setPreviewPicture($pocketEntry['images'][1]['src']); + } + + if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { + $this->contentProxy->assignTagsToEntry( + $entry, + array_keys($pocketEntry['tags']) + ); + } + + $this->em->persist($entry); + ++$this->importedEntries; - if ($this->rabbitMQ) { - foreach ($entries as $entry) { - $this->producer->publish(serialize($entry)); + return $entry; + } + + /** + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + public function parseEntriesForProducer($entries) + { + foreach ($entries as $pocketEntry) { + // set userId for the producer (it won't know which user is connected) + $pocketEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $pocketEntry['status'] = 1; } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($pocketEntry)); } } } -- cgit v1.2.3 From c98db1b653b5dc8b701422190b02d9fbf10c4e68 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 4 Sep 2016 21:49:21 +0200 Subject: Convert other imports to Rabbit --- .../ImportBundle/Import/AbstractImport.php | 83 ++++++++++++++ src/Wallabag/ImportBundle/Import/PocketImport.php | 107 +++-------------- .../ImportBundle/Import/ReadabilityImport.php | 127 +++++++++------------ .../ImportBundle/Import/WallabagImport.php | 118 +++++++------------ .../ImportBundle/Import/WallabagV1Import.php | 20 +++- .../ImportBundle/Import/WallabagV2Import.php | 20 +++- 6 files changed, 235 insertions(+), 240 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 14377a35..5b9d65d7 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -7,12 +7,17 @@ use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; +use Symfony\Component\Security\Core\User\UserInterface; +use OldSound\RabbitMqBundle\RabbitMq\Producer; abstract class AbstractImport implements ImportInterface { protected $em; protected $logger; protected $contentProxy; + protected $producer; + protected $user; + protected $markAsRead; public function __construct(EntityManager $em, ContentProxy $contentProxy) { @@ -26,6 +31,48 @@ abstract class AbstractImport implements ImportInterface $this->logger = $logger; } + /** + * Set RabbitMQ Producer to send each entry to a queue. + * This method should be called when user has enabled RabbitMQ. + * + * @param Producer $producer + */ + public function setRabbitmqProducer(Producer $producer) + { + $this->producer = $producer; + } + + /** + * Set current user. + * Could the current *connected* user or one retrieve by the consumer. + * + * @param UserInterface $user + */ + public function setUser(UserInterface $user) + { + $this->user = $user; + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Get whether articles must be all marked as read. + */ + public function getMarkAsRead() + { + return $this->markAsRead; + } + /** * Fetch content from the ContentProxy (using graby). * If it fails return false instead of the updated entry. @@ -44,4 +91,40 @@ abstract class AbstractImport implements ImportInterface return false; } } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) + { + $i = 1; + + foreach ($entries as $importedEntry) { + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$i; + } + + $this->em->flush(); + } + + /** + * Parse one entry. + * + * @param array $importedEntry + * + * @return Entry + */ + abstract public function parseEntry(array $importedEntry); } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 27df4917..dd0ddd72 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,25 +2,20 @@ namespace Wallabag\ImportBundle\Import; -use OldSound\RabbitMqBundle\RabbitMq\Producer; use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; -use Symfony\Component\Security\Core\User\UserInterface; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Helper\ContentProxy; use Craue\ConfigBundle\Util\Config; class PocketImport extends AbstractImport { - private $user; private $client; private $consumerKey; private $skippedEntries = 0; private $importedEntries = 0; - private $markAsRead; - private $producer; protected $accessToken; public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) @@ -31,28 +26,6 @@ class PocketImport extends AbstractImport $this->logger = new NullLogger(); } - /** - * Set RabbitMQ Producer to send each entry to a queue. - * This method should be called when user has enabled RabbitMQ. - * - * @param Producer $producer - */ - public function setRabbitmqProducer(Producer $producer) - { - $this->producer = $producer; - } - - /** - * Set current user. - * Could the current *connected* user or one retrieve by the consumer. - * - * @param UserInterface $user - */ - public function setUser(UserInterface $user) - { - $this->user = $user; - } - /** * {@inheritdoc} */ @@ -138,26 +111,6 @@ class PocketImport extends AbstractImport return true; } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; - - return $this; - } - - /** - * Get whether articles must be all marked as read. - */ - public function getMarkAsRead() - { - return $this->markAsRead; - } - /** * {@inheritdoc} */ @@ -217,37 +170,9 @@ class PocketImport extends AbstractImport $this->client = $client; } - /** - * @see https://getpocket.com/developer/docs/v3/retrieve - * - * @param array $entries - */ - private function parseEntries(array $entries) - { - $i = 1; - - foreach ($entries as $pocketEntry) { - $entry = $this->parseEntry($pocketEntry); - - if (null === $entry) { - continue; - } - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - $this->em->clear($entry); - } - - ++$i; - } - - $this->em->flush(); - } - - public function parseEntry(array $pocketEntry) + public function parseEntry(array $importedEntry) { - $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; + $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; $existingEntry = $this->em ->getRepository('WallabagCoreBundle:Entry') @@ -270,34 +195,34 @@ class PocketImport extends AbstractImport } // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($pocketEntry['status'] == 1 || $this->markAsRead) { + if ($importedEntry['status'] == 1 || $this->markAsRead) { $entry->setArchived(true); } // 0 or 1 - 1 If the item is starred - if ($pocketEntry['favorite'] == 1) { + if ($importedEntry['favorite'] == 1) { $entry->setStarred(true); } $title = 'Untitled'; - if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { - $title = $pocketEntry['resolved_title']; - } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { - $title = $pocketEntry['given_title']; + if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') { + $title = $importedEntry['resolved_title']; + } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') { + $title = $importedEntry['given_title']; } $entry->setTitle($title); $entry->setUrl($url); // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image - if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { - $entry->setPreviewPicture($pocketEntry['images'][1]['src']); + if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { + $entry->setPreviewPicture($importedEntry['images'][1]['src']); } - if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { + if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { $this->contentProxy->assignTagsToEntry( $entry, - array_keys($pocketEntry['tags']) + array_keys($importedEntry['tags']) ); } @@ -315,17 +240,17 @@ class PocketImport extends AbstractImport */ public function parseEntriesForProducer($entries) { - foreach ($entries as $pocketEntry) { + foreach ($entries as $importedEntry) { // set userId for the producer (it won't know which user is connected) - $pocketEntry['userId'] = $this->user->getId(); + $importedEntry['userId'] = $this->user->getId(); if ($this->markAsRead) { - $pocketEntry['status'] = 1; + $importedEntry['status'] = 1; } ++$this->importedEntries; - $this->producer->publish(json_encode($pocketEntry)); + $this->producer->publish(json_encode($importedEntry)); } } } diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index c7cfe15d..18a6631a 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -7,24 +7,9 @@ use Wallabag\UserBundle\Entity\User; class ReadabilityImport extends AbstractImport { - private $user; private $skippedEntries = 0; private $importedEntries = 0; private $filepath; - private $markAsRead; - - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } /** * {@inheritdoc} @@ -62,26 +47,6 @@ class ReadabilityImport extends AbstractImport return $this; } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; - - return $this; - } - - /** - * Get whether articles must be all marked as read. - */ - public function getMarkAsRead() - { - return $this->markAsRead; - } - /** * {@inheritdoc} */ @@ -116,54 +81,76 @@ class ReadabilityImport extends AbstractImport return false; } + if ($this->producer) { + $this->parseEntriesForProducer($data['bookmarks']); + + return true; + } + $this->parseEntries($data['bookmarks']); return true; } + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $data = [ + 'title' => $importedEntry['article__title'], + 'url' => $importedEntry['article__url'], + 'content_type' => '', + 'language' => '', + 'is_archived' => $importedEntry['archive'] || $this->markAsRead, + 'is_starred' => $importedEntry['favorite'], + ]; + + $entry = $this->fetchContent( + new Entry($this->user), + $data['url'], + $data + ); + + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; + + return; + } + + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + /** - * Parse and insert all given entries. + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. * - * @param $entries + * @param array $entries */ - protected function parseEntries($entries) + protected function parseEntriesForProducer($entries) { - $i = 1; - foreach ($entries as $importedEntry) { - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); - $data = [ - 'title' => $importedEntry['article__title'], - 'url' => $importedEntry['article__url'], - 'content_type' => '', - 'language' => '', - 'is_archived' => $importedEntry['archive'] || $this->markAsRead, - 'is_starred' => $importedEntry['favorite'], - ]; - - $entry = $this->fetchContent( - new Entry($this->user), - $data['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; + if ($this->markAsRead) { + $importedEntry['archive'] = 1; } - $entry->setArchived($data['is_archived']); - $entry->setStarred($data['is_starred']); - $this->em->persist($entry); ++$this->importedEntries; // flush every 20 entries diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 581ec178..6ad14e8c 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -11,7 +11,6 @@ abstract class WallabagImport extends AbstractImport protected $skippedEntries = 0; protected $importedEntries = 0; protected $filepath; - protected $markAsRead; // untitled in all languages from v1 protected $untitled = [ 'Untitled', @@ -28,19 +27,6 @@ abstract class WallabagImport extends AbstractImport '', ]; - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } - /** * {@inheritdoc} */ @@ -79,6 +65,12 @@ abstract class WallabagImport extends AbstractImport return false; } + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); return true; @@ -108,85 +100,61 @@ abstract class WallabagImport extends AbstractImport } /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead + * {@inheritdoc} */ - public function setMarkAsRead($markAsRead) + public function parseEntry(array $importedEntry) { - $this->markAsRead = $markAsRead; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); - return $this; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - /** - * Parse and insert all given entries. - * - * @param $entries - */ - protected function parseEntries($entries) - { - $i = 1; + return; + } + + $data = $this->prepareEntry($importedEntry); - foreach ($entries as $importedEntry) { - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + $entry = $this->fetchContent( + new Entry($this->user), + $importedEntry['url'], + $data + ); - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; - $data = $this->prepareEntry($importedEntry, $this->markAsRead); + return; + } - $entry = $this->fetchContent( - new Entry($this->user), - $importedEntry['url'], - $data + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] ); + } - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; - } - - if (array_key_exists('tags', $data)) { - $this->contentProxy->assignTagsToEntry( - $entry, - $data['tags'] - ); - } - - if (isset($importedEntry['preview_picture'])) { - $entry->setPreviewPicture($importedEntry['preview_picture']); - } - - $entry->setArchived($data['is_archived']); - $entry->setStarred($data['is_starred']); - - $this->em->persist($entry); - ++$this->importedEntries; - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } - ++$i; + if (isset($importedEntry['preview_picture'])) { + $entry->setPreviewPicture($importedEntry['preview_picture']); } - $this->em->flush(); - $this->em->clear(); + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; } /** * This should return a cleaned array for a given entry to be given to `updateEntry`. * - * @param array $entry Data from the imported file - * @param bool $markAsRead Should we mark as read content? + * @param array $entry Data from the imported file * * @return array */ - abstract protected function prepareEntry($entry = [], $markAsRead = false); + abstract protected function prepareEntry($entry = []); } diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 6cf3467a..86734652 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport /** * {@inheritdoc} */ - protected function prepareEntry($entry = [], $markAsRead = false) + protected function prepareEntry($entry = []) { $data = [ 'title' => $entry['title'], @@ -39,7 +39,7 @@ class WallabagV1Import extends WallabagImport 'url' => $entry['url'], 'content_type' => '', 'language' => '', - 'is_archived' => $entry['is_read'] || $markAsRead, + 'is_archived' => $entry['is_read'] || $this->markAsRead, 'is_starred' => $entry['is_fav'], 'tags' => '', ]; @@ -56,4 +56,20 @@ class WallabagV1Import extends WallabagImport return $data; } + + protected function parseEntriesForProducer($entries) + { + foreach ($entries as $importedEntry) { + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry['is_read'] = 1; + } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php index d0035b63..faf4236f 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php @@ -31,12 +31,28 @@ class WallabagV2Import extends WallabagImport /** * {@inheritdoc} */ - protected function prepareEntry($entry = [], $markAsRead = false) + protected function prepareEntry($entry = []) { return [ 'html' => $entry['content'], 'content_type' => $entry['mimetype'], - 'is_archived' => ($entry['is_archived'] || $markAsRead), + 'is_archived' => ($entry['is_archived'] || $this->markAsRead), ] + $entry; } + + protected function parseEntriesForProducer($entries) + { + foreach ($entries as $importedEntry) { + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry['is_archived'] = 1; + } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } } -- cgit v1.2.3 From 02f64895728fe9aee2c696a627e0bbe27a24faf2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 07:13:09 +0200 Subject: Retrieve all items from Pocket 5000 by 5000. Also, retrieve newest item first. --- src/Wallabag/ImportBundle/Import/PocketImport.php | 37 +++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index dd0ddd72..06a31813 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -16,7 +16,9 @@ class PocketImport extends AbstractImport private $consumerKey; private $skippedEntries = 0; private $importedEntries = 0; - protected $accessToken; + private $accessToken; + + const NB_ELEMENTS = 5000; public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) { @@ -26,6 +28,16 @@ class PocketImport extends AbstractImport $this->logger = new NullLogger(); } + /** + * Only used for test purpose + * + * @return string + */ + public function getAccessToken() + { + return $this->accessToken; + } + /** * {@inheritdoc} */ @@ -114,8 +126,10 @@ class PocketImport extends AbstractImport /** * {@inheritdoc} */ - public function import() + public function import($offset = 0) { + static $run = 0; + $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', [ 'body' => json_encode([ @@ -123,7 +137,9 @@ class PocketImport extends AbstractImport 'access_token' => $this->accessToken, 'detailType' => 'complete', 'state' => 'all', - 'sort' => 'oldest', + 'sort' => 'newest', + 'count' => self::NB_ELEMENTS, + 'offset' => $offset, ]), ] ); @@ -140,11 +156,20 @@ class PocketImport extends AbstractImport if ($this->producer) { $this->parseEntriesForProducer($entries['list']); - - return true; + } else { + $this->parseEntries($entries['list']); } - $this->parseEntries($entries['list']); + // if we retrieve exactly the amount of items requested it means we can get more + // re-call import and offset item by the amount previous received: + // - first call get 5k offset 0 + // - second call get 5k offset 5k + // - and so on + if (count($entries['list']) === self::NB_ELEMENTS) { + ++$run; + + return $this->import(self::NB_ELEMENTS * $run); + } return true; } -- cgit v1.2.3 From 3849a9f3231c0109c87af085452c3ac5e4aed303 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 07:50:10 +0200 Subject: Some cleanup & refactor --- .../ImportBundle/Import/AbstractImport.php | 42 ++++++++++++++++++++-- src/Wallabag/ImportBundle/Import/PocketImport.php | 22 +++--------- .../ImportBundle/Import/ReadabilityImport.php | 28 +++------------ .../ImportBundle/Import/WallabagImport.php | 2 -- .../ImportBundle/Import/WallabagV1Import.php | 18 ++++------ .../ImportBundle/Import/WallabagV2Import.php | 18 ++++------ 6 files changed, 60 insertions(+), 70 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 5b9d65d7..b085dc3a 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -7,7 +7,7 @@ use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; -use Symfony\Component\Security\Core\User\UserInterface; +use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\Producer; abstract class AbstractImport implements ImportInterface @@ -46,9 +46,9 @@ abstract class AbstractImport implements ImportInterface * Set current user. * Could the current *connected* user or one retrieve by the consumer. * - * @param UserInterface $user + * @param User $user */ - public function setUser(UserInterface $user) + public function setUser(User $user) { $this->user = $user; } @@ -119,6 +119,32 @@ abstract class AbstractImport implements ImportInterface $this->em->flush(); } + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) + { + foreach ($entries as $importedEntry) { + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } + /** * Parse one entry. * @@ -127,4 +153,14 @@ abstract class AbstractImport implements ImportInterface * @return Entry */ abstract public function parseEntry(array $importedEntry); + + /** + * Set current imported entry to archived / read. + * Implementation is different accross all imports. + * + * @param array $importedEntry + * + * @return array + */ + abstract protected function setEntryAsRead(array $importedEntry); } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 06a31813..5850deba 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -29,7 +29,7 @@ class PocketImport extends AbstractImport } /** - * Only used for test purpose + * Only used for test purpose. * * @return string */ @@ -258,24 +258,12 @@ class PocketImport extends AbstractImport } /** - * Faster parse entries for Producer. - * We don't care to make check at this time. They'll be done by the consumer. - * - * @param array $entries + * {@inheritdoc} */ - public function parseEntriesForProducer($entries) + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); - - if ($this->markAsRead) { - $importedEntry['status'] = 1; - } + $importedEntry['status'] = 1; - ++$this->importedEntries; - - $this->producer->publish(json_encode($importedEntry)); - } + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 18a6631a..64ef62bf 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -3,7 +3,6 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; class ReadabilityImport extends AbstractImport { @@ -136,31 +135,12 @@ class ReadabilityImport extends AbstractImport } /** - * Faster parse entries for Producer. - * We don't care to make check at this time. They'll be done by the consumer. - * - * @param array $entries + * {@inheritdoc} */ - protected function parseEntriesForProducer($entries) + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); - - if ($this->markAsRead) { - $importedEntry['archive'] = 1; - } - - ++$this->importedEntries; - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } - ++$i; - } + $importedEntry['archive'] = 1; - $this->em->flush(); - $this->em->clear(); + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 6ad14e8c..8e18e0ef 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -3,11 +3,9 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; abstract class WallabagImport extends AbstractImport { - protected $user; protected $skippedEntries = 0; protected $importedEntries = 0; protected $filepath; diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 86734652..292b72a7 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -57,19 +57,13 @@ class WallabagV1Import extends WallabagImport return $data; } - protected function parseEntriesForProducer($entries) + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); + $importedEntry['is_read'] = 1; - if ($this->markAsRead) { - $importedEntry['is_read'] = 1; - } - - ++$this->importedEntries; - - $this->producer->publish(json_encode($importedEntry)); - } + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php index faf4236f..37c8ca14 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php @@ -40,19 +40,13 @@ class WallabagV2Import extends WallabagImport ] + $entry; } - protected function parseEntriesForProducer($entries) + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); - - if ($this->markAsRead) { - $importedEntry['is_archived'] = 1; - } - - ++$this->importedEntries; + $importedEntry['is_archived'] = 1; - $this->producer->publish(json_encode($importedEntry)); - } + return $importedEntry; } } -- cgit v1.2.3 From 3aca0a9f00417b64203a660dee0a2b4c0fe22ac8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 09:35:42 +0200 Subject: CS --- src/Wallabag/ImportBundle/Import/AbstractImport.php | 2 ++ src/Wallabag/ImportBundle/Import/PocketImport.php | 2 -- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 2 -- src/Wallabag/ImportBundle/Import/WallabagImport.php | 2 -- 4 files changed, 2 insertions(+), 6 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index b085dc3a..8610062d 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -18,6 +18,8 @@ abstract class AbstractImport implements ImportInterface protected $producer; protected $user; protected $markAsRead; + protected $skippedEntries = 0; + protected $importedEntries = 0; public function __construct(EntityManager $em, ContentProxy $contentProxy) { diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 5850deba..845380b7 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -14,8 +14,6 @@ class PocketImport extends AbstractImport { private $client; private $consumerKey; - private $skippedEntries = 0; - private $importedEntries = 0; private $accessToken; const NB_ELEMENTS = 5000; diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 64ef62bf..915d4cd3 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -6,8 +6,6 @@ use Wallabag\CoreBundle\Entity\Entry; class ReadabilityImport extends AbstractImport { - private $skippedEntries = 0; - private $importedEntries = 0; private $filepath; /** diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 8e18e0ef..026567b0 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -6,8 +6,6 @@ use Wallabag\CoreBundle\Entity\Entry; abstract class WallabagImport extends AbstractImport { - protected $skippedEntries = 0; - protected $importedEntries = 0; protected $filepath; // untitled in all languages from v1 protected $untitled = [ -- cgit v1.2.3 From 6d65c0a8b089d3caa6f8e20d7935a9fe2f87d926 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 09:36:07 +0200 Subject: Add ability to define created_at for all import At the moment only Readability & wallabag v2 import allow created_at import. Pocket removed `time_added` field from their API v2 to v3... And wallabag v1 doesn't export that value. --- src/Wallabag/ImportBundle/Import/PocketImport.php | 5 +++++ src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 5 +++++ src/Wallabag/ImportBundle/Import/WallabagImport.php | 4 ++++ src/Wallabag/ImportBundle/Import/WallabagV1Import.php | 1 + 4 files changed, 15 insertions(+) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 845380b7..92dcdd40 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -193,6 +193,11 @@ class PocketImport extends AbstractImport $this->client = $client; } + /** + * {@inheritdoc} + * + * @see https://getpocket.com/developer/docs/v3/retrieve + */ public function parseEntry(array $importedEntry) { $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 915d4cd3..8f080d38 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -89,6 +89,9 @@ class ReadabilityImport extends AbstractImport return true; } + /** + * {@inheritdoc} + */ public function parseEntry(array $importedEntry) { $existingEntry = $this->em @@ -108,6 +111,7 @@ class ReadabilityImport extends AbstractImport 'language' => '', 'is_archived' => $importedEntry['archive'] || $this->markAsRead, 'is_starred' => $importedEntry['favorite'], + 'created_at' => $importedEntry['date_added'], ]; $entry = $this->fetchContent( @@ -125,6 +129,7 @@ class ReadabilityImport extends AbstractImport $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); + $entry->setCreatedAt(new \DateTime($data['created_at'])); $this->em->persist($entry); ++$this->importedEntries; diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 026567b0..8e50b135 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -139,6 +139,10 @@ abstract class WallabagImport extends AbstractImport $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); + if (!empty($data['created_at'])) { + $entry->setCreatedAt(new \DateTime($data['created_at'])); + } + $this->em->persist($entry); ++$this->importedEntries; diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 292b72a7..4f001062 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -42,6 +42,7 @@ class WallabagV1Import extends WallabagImport 'is_archived' => $entry['is_read'] || $this->markAsRead, 'is_starred' => $entry['is_fav'], 'tags' => '', + 'created_at' => '', ]; // force content to be refreshed in case on bad fetch in the v1 installation -- cgit v1.2.3 From 8664069e1aa2fa89e17587308a03f2720c20327a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 10:12:25 +0200 Subject: Fix DateTime & clear() --- src/Wallabag/ImportBundle/Import/AbstractImport.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 8610062d..39befa7b 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -7,6 +7,7 @@ use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\Producer; @@ -113,7 +114,10 @@ abstract class AbstractImport implements ImportInterface // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } ++$i; } -- cgit v1.2.3 From 13470c3596d0b1490bbf18b39128a05bbb3c7f3e Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 18:02:29 +0200 Subject: Add test for RabbitMQ Also update Symfony deps --- src/Wallabag/ImportBundle/Import/PocketImport.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 92dcdd40..d76a3a08 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -265,7 +265,7 @@ class PocketImport extends AbstractImport */ protected function setEntryAsRead(array $importedEntry) { - $importedEntry['status'] = 1; + $importedEntry['status'] = '1'; return $importedEntry; } -- cgit v1.2.3 From 7f7531171f6e49110b5842f869e37c766a682473 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 20:45:30 +0200 Subject: Retrieve created date from Pocket --- src/Wallabag/ImportBundle/Import/PocketImport.php | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index d76a3a08..fe39d33f 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -254,6 +254,10 @@ class PocketImport extends AbstractImport ); } + if (!empty($importedEntry['time_added'])) { + $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added'])); + } + $this->em->persist($entry); ++$this->importedEntries; -- cgit v1.2.3 From b3437d58ae224121375c99e9288d8b808524e624 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 21:02:03 +0200 Subject: Enable Redis async import - using javibravo/simpleue - internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear - if both option are enable rabbit will be choosen - services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml` - --- src/Wallabag/ImportBundle/Import/AbstractImport.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 39befa7b..4cd8e846 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -9,7 +9,7 @@ use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; -use OldSound\RabbitMqBundle\RabbitMq\Producer; +use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; abstract class AbstractImport implements ImportInterface { @@ -35,12 +35,12 @@ abstract class AbstractImport implements ImportInterface } /** - * Set RabbitMQ Producer to send each entry to a queue. + * Set RabbitMQ/Redis Producer to send each entry to a queue. * This method should be called when user has enabled RabbitMQ. * - * @param Producer $producer + * @param ProducerInterface $producer */ - public function setRabbitmqProducer(Producer $producer) + public function setProducer(ProducerInterface $producer) { $this->producer = $producer; } -- cgit v1.2.3 From c80cc01afa315dcfa38b2a01c5b05d4516659c24 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 21:09:05 +0200 Subject: Change flash message for queued articles --- src/Wallabag/ImportBundle/Import/AbstractImport.php | 15 ++++++++++++++- src/Wallabag/ImportBundle/Import/PocketImport.php | 11 ----------- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 11 ----------- src/Wallabag/ImportBundle/Import/WallabagImport.php | 11 ----------- 4 files changed, 14 insertions(+), 34 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 4cd8e846..2af0e69b 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -21,6 +21,7 @@ abstract class AbstractImport implements ImportInterface protected $markAsRead; protected $skippedEntries = 0; protected $importedEntries = 0; + protected $queuedEntries = 0; public function __construct(EntityManager $em, ContentProxy $contentProxy) { @@ -145,12 +146,24 @@ abstract class AbstractImport implements ImportInterface $importedEntry = $this->setEntryAsRead($importedEntry); } - ++$this->importedEntries; + ++$this->queuedEntries; $this->producer->publish(json_encode($importedEntry)); } } + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + 'queued' => $this->queuedEntries, + ]; + } + /** * Parse one entry. * diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index fe39d33f..cc6faf1f 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -172,17 +172,6 @@ class PocketImport extends AbstractImport return true; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * Set the Guzzle client. * diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 8f080d38..b852f8f0 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -44,17 +44,6 @@ class ReadabilityImport extends AbstractImport return $this; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 8e50b135..969a6a04 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -72,17 +72,6 @@ abstract class WallabagImport extends AbstractImport return true; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * Set file path to the json file. * -- cgit v1.2.3 From ebe0787e093f4f2934430033015d6ebad1c64dca Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 22:22:25 +0200 Subject: Moved Pocket token to user config --- src/Wallabag/ImportBundle/Import/PocketImport.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index cc6faf1f..40603c90 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -13,16 +13,14 @@ use Craue\ConfigBundle\Util\Config; class PocketImport extends AbstractImport { private $client; - private $consumerKey; private $accessToken; const NB_ELEMENTS = 5000; - public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) + public function __construct(EntityManager $em, ContentProxy $contentProxy) { $this->em = $em; $this->contentProxy = $contentProxy; - $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); } @@ -72,7 +70,7 @@ class PocketImport extends AbstractImport $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'redirect_uri' => $redirectUri, ]), ] @@ -102,7 +100,7 @@ class PocketImport extends AbstractImport $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'code' => $code, ]), ] @@ -131,7 +129,7 @@ class PocketImport extends AbstractImport $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'access_token' => $this->accessToken, 'detailType' => 'complete', 'state' => 'all', -- cgit v1.2.3 From fbb319f064e6336a3b44bda12cdc51c93c51f379 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 22:58:33 +0200 Subject: Missing some migrations and CS --- src/Wallabag/ImportBundle/Import/PocketImport.php | 1 - 1 file changed, 1 deletion(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 40603c90..1bf22d68 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -8,7 +8,6 @@ use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Helper\ContentProxy; -use Craue\ConfigBundle\Util\Config; class PocketImport extends AbstractImport { -- cgit v1.2.3 From 59b97fae996d8307b9d957d210d46200f6d206bf Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 17 Sep 2016 07:40:56 +0200 Subject: Avoid losing entry when fetching fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of just say “Failed to save entry” we’ll save the entry at all cost and try to fetch content. If fetching content failed, the entry will still be saved at least, but without content. --- src/Wallabag/ImportBundle/Import/AbstractImport.php | 6 +++--- src/Wallabag/ImportBundle/Import/PocketImport.php | 19 +++++-------------- .../ImportBundle/Import/ReadabilityImport.php | 16 +++++----------- src/Wallabag/ImportBundle/Import/WallabagImport.php | 16 +++++----------- 4 files changed, 18 insertions(+), 39 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 2af0e69b..a1a14576 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -79,20 +79,20 @@ abstract class AbstractImport implements ImportInterface /** * Fetch content from the ContentProxy (using graby). - * If it fails return false instead of the updated entry. + * If it fails return the given entry to be saved in all case (to avoid user to loose the content). * * @param Entry $entry Entry to update * @param string $url Url to grab content for * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url * - * @return Entry|false + * @return Entry */ protected function fetchContent(Entry $entry, $url, array $content = []) { try { return $this->contentProxy->updateEntry($entry, $url, $content); } catch (\Exception $e) { - return false; + return $entry; } } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 1bf22d68..e00eb44b 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -199,24 +199,16 @@ class PocketImport extends AbstractImport } $entry = new Entry($this->user); - $entry = $this->fetchContent($entry, $url); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry->setUrl($url); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $url); // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($importedEntry['status'] == 1 || $this->markAsRead) { - $entry->setArchived(true); - } + $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead); // 0 or 1 - 1 If the item is starred - if ($importedEntry['favorite'] == 1) { - $entry->setStarred(true); - } + $entry->setStarred($importedEntry['favorite'] == 1); $title = 'Untitled'; if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') { @@ -226,7 +218,6 @@ class PocketImport extends AbstractImport } $entry->setTitle($title); - $entry->setUrl($url); // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index b852f8f0..fa2b7053 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -103,18 +103,12 @@ class ReadabilityImport extends AbstractImport 'created_at' => $importedEntry['date_added'], ]; - $entry = $this->fetchContent( - new Entry($this->user), - $data['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 969a6a04..043bb0a2 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -101,18 +101,12 @@ abstract class WallabagImport extends AbstractImport $data = $this->prepareEntry($importedEntry); - $entry = $this->fetchContent( - new Entry($this->user), - $importedEntry['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); if (array_key_exists('tags', $data)) { $this->contentProxy->assignTagsToEntry( -- cgit v1.2.3 From 401135852c6b25c8d5ab97beaefb02d1bd023ec9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 25 Sep 2016 11:26:15 +0200 Subject: Use scheduled entity insertions to avoid tag duplicate Using `getScheduledEntityInsertions()` we can retrieve not yet flushed but already persisted entities and then avoid tags duplication on import. --- src/Wallabag/ImportBundle/Import/PocketImport.php | 3 ++- src/Wallabag/ImportBundle/Import/WallabagImport.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index e00eb44b..327e2500 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -227,7 +227,8 @@ class PocketImport extends AbstractImport if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { $this->contentProxy->assignTagsToEntry( $entry, - array_keys($importedEntry['tags']) + array_keys($importedEntry['tags']), + $this->em->getUnitOfWork()->getScheduledEntityInsertions() ); } diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 043bb0a2..3754e4a9 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -111,7 +111,8 @@ abstract class WallabagImport extends AbstractImport if (array_key_exists('tags', $data)) { $this->contentProxy->assignTagsToEntry( $entry, - $data['tags'] + $data['tags'], + $this->em->getUnitOfWork()->getScheduledEntityInsertions() ); } -- cgit v1.2.3 From ae669126e718ede5dbf76929215d8514cd960976 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 12 Jul 2016 13:51:05 +0200 Subject: Import Firefox & Chrome bookmarks into wallabag --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 227 +++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 src/Wallabag/ImportBundle/Import/BrowserImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php new file mode 100644 index 00000000..263a11d5 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -0,0 +1,227 @@ +em = $em; + $this->logger = new NullLogger(); + $this->contentProxy = $contentProxy; + } + + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * We define the user in a custom call because on the import command there is no logged in user. + * So we can't retrieve user from the `security.token_storage` service. + * + * @param User $user + * + * @return $this + */ + public function setUser(User $user) + { + $this->user = $user; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'Firefox & Google Chrome'; + } + + /** + * {@inheritdoc} + */ + public function getUrl() + { + return 'import_browser'; + } + + /** + * {@inheritdoc} + */ + public function getDescription() + { + return 'import.browser.description'; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('WallabagImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('WallabagImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + return false; + } + + $this->nbEntries = 1; + $this->parseEntries($data); + $this->em->flush(); + + return true; + } + + private function parseEntries($data) + { + foreach ($data as $importedEntry) { + $this->parseEntry($importedEntry); + } + $this->totalEntries += count($data); + } + + private function parseEntry($importedEntry) + { + if (!is_array($importedEntry)) { + return; + } + + /* Firefox uses guid while Chrome uses id */ + + if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { + $this->parseEntries($importedEntry); + + return; + } + if (key_exists('children', $importedEntry)) { + $this->parseEntries($importedEntry['children']); + + return; + } + if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { + + /* Firefox uses uri while Chrome uses url */ + + $firefox = key_exists('uri', $importedEntry); + + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { + $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); + ++$this->skippedEntries; + + return; + } + + try { + $entry = $this->contentProxy->updateEntry( + new Entry($this->user), + ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] + ); + } catch (\Exception $e) { + $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); + ++$this->skippedEntries; + + return; + } + + $entry->setArchived($this->markAsRead); + + $this->em->persist($entry); + ++$this->importedEntries; + + // flush every 20 entries + if (($this->nbEntries % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$this->nbEntries; + + /* + + Maybe not useful. Delete at will. + + */ + + $this->logger->info($this->nbEntries.' / '.$this->totalEntries); + } + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + * + * @return $this + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Set file path to the json file. + * + * @param string $filepath + * + * @return $this + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + ]; + } +} -- cgit v1.2.3 From efe659ab84df5db23d95203c0cef8c43ed0914e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 2 Sep 2016 13:53:45 +0200 Subject: Add Chrome path for Mac OS --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 8 -------- 1 file changed, 8 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 263a11d5..e3457196 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -175,14 +175,6 @@ class BrowserImport implements ImportInterface $this->em->clear($entry); } ++$this->nbEntries; - - /* - - Maybe not useful. Delete at will. - - */ - - $this->logger->info($this->nbEntries.' / '.$this->totalEntries); } } -- cgit v1.2.3 From 59201088b4fc13fd361238396f630dabd9bd1990 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 17:47:47 +0200 Subject: bring chrome and firefox as separate imports --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 247 ++++++++++----------- src/Wallabag/ImportBundle/Import/ChromeImport.php | 71 ++++++ src/Wallabag/ImportBundle/Import/FirefoxImport.php | 71 ++++++ 3 files changed, 262 insertions(+), 127 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Import/ChromeImport.php create mode 100644 src/Wallabag/ImportBundle/Import/FirefoxImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index e3457196..ef7d6d95 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -9,69 +9,24 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; -class BrowserImport implements ImportInterface +abstract class BrowserImport extends AbstractImport { - protected $user; - protected $em; - protected $logger; - protected $contentProxy; - protected $skippedEntries = 0; - protected $importedEntries = 0; - protected $totalEntries = 0; protected $filepath; - protected $markAsRead; - private $nbEntries; - - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->logger = new NullLogger(); - $this->contentProxy = $contentProxy; - } - - public function setLogger(LoggerInterface $logger) - { - $this->logger = $logger; - } - - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - * - * @return $this - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } /** * {@inheritdoc} */ - public function getName() - { - return 'Firefox & Google Chrome'; - } + abstract public function getName(); /** * {@inheritdoc} */ - public function getUrl() - { - return 'import_browser'; - } + abstract public function getUrl(); /** * {@inheritdoc} */ - public function getDescription() - { - return 'import.browser.description'; - } + abstract public function getDescription(); /** * {@inheritdoc} @@ -96,124 +51,162 @@ class BrowserImport implements ImportInterface return false; } - $this->nbEntries = 1; + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); - $this->em->flush(); return true; } - private function parseEntries($data) + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) { - foreach ($data as $importedEntry) { - $this->parseEntry($importedEntry); + $i = 1; + + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } + ++$i; } - $this->totalEntries += count($data); + + $this->em->flush(); } - private function parseEntry($importedEntry) + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) { - if (!is_array($importedEntry)) { - return; + foreach ($entries as $importedEntry) { + + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); } + } - /* Firefox uses guid while Chrome uses id */ + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); - return; } + if (key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); - return; } - if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { - - /* Firefox uses uri while Chrome uses url */ - $firefox = key_exists('uri', $importedEntry); - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; + if (!key_exists('uri', $importedEntry) && !key_exists('url', $importedEntry)) { + return; + } - return; - } + $firefox = key_exists('uri', $importedEntry); - if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { - $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); - ++$this->skippedEntries; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - return; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - try { - $entry = $this->contentProxy->updateEntry( - new Entry($this->user), - ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] - ); - } catch (\Exception $e) { - $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); - ++$this->skippedEntries; + return; + } - return; - } + $data = $this->prepareEntry($importedEntry); - $entry->setArchived($this->markAsRead); + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - $this->em->persist($entry); - ++$this->importedEntries; + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); - // flush every 20 entries - if (($this->nbEntries % 20) === 0) { - $this->em->flush(); - $this->em->clear($entry); - } - ++$this->nbEntries; + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] + ); } - } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - * - * @return $this - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; + $entry->setArchived($data['is_archived']); - return $this; - } + if (!empty($data['created_at'])) { + $dt = new \DateTime(); + $entry->setCreatedAt($dt->setTimestamp($data['created_at']/1000)); + } - /** - * Set file path to the json file. - * - * @param string $filepath - * - * @return $this - */ - public function setFilepath($filepath) - { - $this->filepath = $filepath; + $this->em->persist($entry); + ++$this->importedEntries; - return $this; + return $entry; } /** * {@inheritdoc} */ - public function getSummary() + protected function setEntryAsRead(array $importedEntry) { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; + $importedEntry['is_archived'] = 1; + + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php new file mode 100644 index 00000000..7936ee2f --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php new file mode 100644 index 00000000..cbf10b87 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} -- cgit v1.2.3 From 2c61db30b737685ae9102ec10f2371778fb13f1a Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 18:00:08 +0200 Subject: cs & fixes --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 9 +++------ src/Wallabag/ImportBundle/Import/ChromeImport.php | 10 +--------- src/Wallabag/ImportBundle/Import/FirefoxImport.php | 20 ++++++-------------- 3 files changed, 10 insertions(+), 29 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index ef7d6d95..3e1cb12b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -2,9 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; @@ -121,7 +118,6 @@ abstract class BrowserImport extends AbstractImport protected function parseEntriesForProducer(array $entries) { foreach ($entries as $importedEntry) { - if ((array) $importedEntry !== $importedEntry) { continue; } @@ -144,14 +140,15 @@ abstract class BrowserImport extends AbstractImport */ public function parseEntry(array $importedEntry) { - if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); + return; } if (key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); + return; } @@ -191,7 +188,7 @@ abstract class BrowserImport extends AbstractImport if (!empty($data['created_at'])) { $dt = new \DateTime(); - $entry->setCreatedAt($dt->setTimestamp($data['created_at']/1000)); + $entry->setCreatedAt($dt->setTimestamp($data['created_at'] / 1000)); } $this->em->persist($entry); diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 7936ee2f..941d68d3 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -2,13 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; -use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; -use Wallabag\CoreBundle\Helper\ContentProxy; - class ChromeImport extends BrowserImport { protected $filepath; @@ -37,7 +30,7 @@ class ChromeImport extends BrowserImport return 'import.chrome.description'; } - /** + /** * {@inheritdoc} */ protected function prepareEntry($entry = []) @@ -58,7 +51,6 @@ class ChromeImport extends BrowserImport return $data; } - /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index cbf10b87..351cbef1 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -2,13 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; -use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; -use Wallabag\CoreBundle\Helper\ContentProxy; - class FirefoxImport extends BrowserImport { protected $filepath; @@ -42,7 +35,7 @@ class FirefoxImport extends BrowserImport */ protected function prepareEntry($entry = []) { - $data = [ + $data = [ 'title' => $entry['name'], 'html' => '', 'url' => $entry['url'], @@ -51,13 +44,12 @@ class FirefoxImport extends BrowserImport 'created_at' => $entry['date_added'], ]; - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + return $data; + } /** * {@inheritdoc} -- cgit v1.2.3 From 64b1229b2d711e6b2f0e60de482802d9e86b912f Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 19:24:19 +0200 Subject: fix tests --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 4 +-- src/Wallabag/ImportBundle/Import/ChromeImport.php | 40 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 3e1cb12b..198e148e 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -31,13 +31,13 @@ abstract class BrowserImport extends AbstractImport public function import() { if (!$this->user) { - $this->logger->error('WallabagImport: user is not defined'); + $this->logger->error('Wallabag Browser Import: user is not defined'); return false; } if (!file_exists($this->filepath) || !is_readable($this->filepath)) { - $this->logger->error('WallabagImport: unable to read file', ['filepath' => $this->filepath]); + $this->logger->error('Wallabag Browser Import: unable to read file', ['filepath' => $this->filepath]); return false; } diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 941d68d3..1af7cc87 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -30,26 +30,26 @@ class ChromeImport extends BrowserImport return 'import.chrome.description'; } - /** - * {@inheritdoc} - */ - protected function prepareEntry($entry = []) - { - $data = [ - 'title' => $entry['name'], - 'html' => '', - 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $entry['date_added'], - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } + /** + * {@inheritdoc} + */ + protected function prepareEntry($entry = []) + { + $data = [ + 'title' => $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } /** * {@inheritdoc} -- cgit v1.2.3 From bd206a84d8c333c1287f5cb81d2e29f3afe515c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 23 Sep 2016 10:56:08 +0200 Subject: Fixed tests by removing clear() --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 198e148e..44315e8b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -94,10 +94,6 @@ abstract class BrowserImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); } ++$i; } -- cgit v1.2.3 From 27acc6ddb8b0a1549a3f015171621e3056ef65d2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 25 Sep 2016 15:29:40 +0200 Subject: Fix bad date format in Browser import --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 23 ++++++++++++++++ src/Wallabag/ImportBundle/Import/ChromeImport.php | 31 ---------------------- src/Wallabag/ImportBundle/Import/FirefoxImport.php | 31 ---------------------- 3 files changed, 23 insertions(+), 62 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 44315e8b..da69df9b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -193,6 +193,29 @@ abstract class BrowserImport extends AbstractImport return $entry; } + /** + * {@inheritdoc} + */ + protected function prepareEntry($entry = []) + { + $data = [ + 'title' => $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + // date are in format like "13118829474385693" + // and it'll be devided by 1000 in AbstractImport + 'created_at' => (int) ceil($entry['date_added'] / 10000), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 1af7cc87..60602a1b 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -29,35 +29,4 @@ class ChromeImport extends BrowserImport { return 'import.chrome.description'; } - - /** - * {@inheritdoc} - */ - protected function prepareEntry($entry = []) - { - $data = [ - 'title' => $entry['name'], - 'html' => '', - 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $entry['date_added'], - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } - - /** - * {@inheritdoc} - */ - protected function setEntryAsRead(array $importedEntry) - { - $importedEntry['is_archived'] = 1; - - return $importedEntry; - } } diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index 351cbef1..1a0b1154 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -29,35 +29,4 @@ class FirefoxImport extends BrowserImport { return 'import.firefox.description'; } - - /** - * {@inheritdoc} - */ - protected function prepareEntry($entry = []) - { - $data = [ - 'title' => $entry['name'], - 'html' => '', - 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $entry['date_added'], - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } - - /** - * {@inheritdoc} - */ - protected function setEntryAsRead(array $importedEntry) - { - $importedEntry['is_archived'] = 1; - - return $importedEntry; - } } -- cgit v1.2.3 From 12d93e6896f2d99b6329b7979ee7b6d11e457c3a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 25 Sep 2016 22:24:07 +0200 Subject: Update Firefox file With real data, the previous looks more than a Chrome converted file. Also, fix date conversion (hope so). --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 36 ++++++++++++++-------- 1 file changed, 24 insertions(+), 12 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index da69df9b..68fa8bf8 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -136,27 +136,27 @@ abstract class BrowserImport extends AbstractImport */ public function parseEntry(array $importedEntry) { - if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { + if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); return; } - if (key_exists('children', $importedEntry)) { + if (array_key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); return; } - if (!key_exists('uri', $importedEntry) && !key_exists('url', $importedEntry)) { + if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) { return; } - $firefox = key_exists('uri', $importedEntry); + $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url']; $existingEntry = $this->em ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); + ->findByUrlAndUserId($url, $this->user->getId()); if (false !== $existingEntry) { ++$this->skippedEntries; @@ -184,7 +184,7 @@ abstract class BrowserImport extends AbstractImport if (!empty($data['created_at'])) { $dt = new \DateTime(); - $entry->setCreatedAt($dt->setTimestamp($data['created_at'] / 1000)); + $entry->setCreatedAt($dt->setTimestamp($data['created_at'])); } $this->em->persist($entry); @@ -196,17 +196,29 @@ abstract class BrowserImport extends AbstractImport /** * {@inheritdoc} */ - protected function prepareEntry($entry = []) + protected function prepareEntry(array $entry = []) { + $url = array_key_exists('uri', $entry) ? $entry['uri'] : $entry['url']; + $date = array_key_exists('date_added', $entry) ? $entry['date_added'] : $entry['dateAdded']; + $title = array_key_exists('name', $entry) ? $entry['name'] : $entry['title']; + + if (16 === strlen($date)) { + // firefox ... + $date = (int) ceil($date / 1000000); + } else if (17 === strlen($date)) { + // chrome ... + $date = (int) ceil($date / 10000000); + } else { + $date = ''; + } + $data = [ - 'title' => $entry['name'], + 'title' => $title, 'html' => '', - 'url' => $entry['url'], + 'url' => $url, 'is_archived' => $this->markAsRead, 'tags' => '', - // date are in format like "13118829474385693" - // and it'll be devided by 1000 in AbstractImport - 'created_at' => (int) ceil($entry['date_added'] / 10000), + 'created_at' => $date, ]; if (array_key_exists('tags', $entry) && $entry['tags'] != '') { -- cgit v1.2.3 From 990adfb34c148e7cd28b9cb784cf2b7388caae8f Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 26 Sep 2016 07:30:02 +0200 Subject: Move prepareEntry to dedicated place Yeah first try was ugly, now each part are in the dedicated place. Also, the date is hardly truncated to 10 chars because Firefox date are 16 chars long and Chrome are 17 chars long. So instead of divised them by a huge number, I prefer to truncate them. --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 35 ---------------------- src/Wallabag/ImportBundle/Import/ChromeImport.php | 21 +++++++++++++ src/Wallabag/ImportBundle/Import/FirefoxImport.php | 21 +++++++++++++ 3 files changed, 42 insertions(+), 35 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 68fa8bf8..e15443c4 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -193,41 +193,6 @@ abstract class BrowserImport extends AbstractImport return $entry; } - /** - * {@inheritdoc} - */ - protected function prepareEntry(array $entry = []) - { - $url = array_key_exists('uri', $entry) ? $entry['uri'] : $entry['url']; - $date = array_key_exists('date_added', $entry) ? $entry['date_added'] : $entry['dateAdded']; - $title = array_key_exists('name', $entry) ? $entry['name'] : $entry['title']; - - if (16 === strlen($date)) { - // firefox ... - $date = (int) ceil($date / 1000000); - } else if (17 === strlen($date)) { - // chrome ... - $date = (int) ceil($date / 10000000); - } else { - $date = ''; - } - - $data = [ - 'title' => $title, - 'html' => '', - 'url' => $url, - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $date, - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } - /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 60602a1b..d7620bcb 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -29,4 +29,25 @@ class ChromeImport extends BrowserImport { return 'import.chrome.description'; } + + /** + * {@inheritdoc} + */ + protected function prepareEntry(array $entry = []) + { + $data = [ + 'title' => $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => substr($entry['date_added'], 0, 10), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } } diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index 1a0b1154..e010f5a4 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -29,4 +29,25 @@ class FirefoxImport extends BrowserImport { return 'import.firefox.description'; } + + /** + * {@inheritdoc} + */ + protected function prepareEntry(array $entry = []) + { + $data = [ + 'title' => $entry['title'], + 'html' => '', + 'url' => $entry['uri'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => substr($entry['dateAdded'], 0, 10), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } } -- cgit v1.2.3 From ff1a5362f7254d686864ea53994da6c517b3d3e8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 27 Sep 2016 07:57:53 +0200 Subject: Add Instapaper import Also update ImportController with latest import (chrome, firefox & instapaper). --- .../ImportBundle/Import/AbstractImport.php | 4 + .../ImportBundle/Import/InstapaperImport.php | 134 +++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 src/Wallabag/ImportBundle/Import/InstapaperImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index a1a14576..764b390a 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -106,6 +106,10 @@ abstract class AbstractImport implements ImportInterface $i = 1; foreach ($entries as $importedEntry) { + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + $entry = $this->parseEntry($importedEntry); if (null === $entry) { diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php new file mode 100644 index 00000000..356acf23 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php @@ -0,0 +1,134 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('InstapaperImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('InstapaperImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $entries = []; + $handle = fopen($this->filepath, 'r'); + while (($data = fgetcsv($handle, 10240)) !== false) { + if ('URL' === $data[0]) { + continue; + } + + $entries[] = [ + 'url' => $data[0], + 'title' => $data[1], + 'status' => $data[3], + 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred', + 'is_starred' => $data[3] === 'Starred', + 'content_type' => '', + 'language' => '', + ]; + } + fclose($handle); + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $entry = new Entry($this->user); + $entry->setUrl($importedEntry['url']); + $entry->setTitle($importedEntry['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry); + + $entry->setArchived($importedEntry['is_archived']); + $entry->setStarred($importedEntry['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} -- cgit v1.2.3 From c7ea9b41f32f222fef6a59734ea0b1176bfa1f41 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 27 Sep 2016 17:01:14 +0200 Subject: Add controller test for Instapaper --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 2 ++ src/Wallabag/ImportBundle/Import/InstapaperImport.php | 6 ++++++ src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 2 ++ src/Wallabag/ImportBundle/Import/WallabagImport.php | 2 ++ 4 files changed, 12 insertions(+) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index e15443c4..9d75685b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -45,6 +45,8 @@ abstract class BrowserImport extends AbstractImport $data = json_decode(file_get_contents($this->filepath), true); if (empty($data)) { + $this->logger->error('Wallabag Browser: no entries in imported file'); + return false; } diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php index 356acf23..cf4c785c 100644 --- a/src/Wallabag/ImportBundle/Import/InstapaperImport.php +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php @@ -80,6 +80,12 @@ class InstapaperImport extends AbstractImport } fclose($handle); + if (empty($entries)) { + $this->logger->error('InstapaperImport: no entries in imported file'); + + return false; + } + if ($this->producer) { $this->parseEntriesForProducer($entries); diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index fa2b7053..b8c0f777 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -64,6 +64,8 @@ class ReadabilityImport extends AbstractImport $data = json_decode(file_get_contents($this->filepath), true); if (empty($data) || empty($data['bookmarks'])) { + $this->logger->error('ReadabilityImport: no entries in imported file'); + return false; } diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 3754e4a9..702da057 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -58,6 +58,8 @@ abstract class WallabagImport extends AbstractImport $data = json_decode(file_get_contents($this->filepath), true); if (empty($data)) { + $this->logger->error('WallabagImport: no entries in imported file'); + return false; } -- cgit v1.2.3