From 58fadbc9df3f8b735c04995919b6cf913ca6a977 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 15:53:16 +0200 Subject: Fix error on EntityManager clear Introduced in the recent 2.5.5 release. Also updated deps. --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index a1cc085b..9cd3dcb8 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -172,7 +172,10 @@ abstract class WallabagImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } ++$i; } -- cgit v1.2.3 From 88e88016b9a86d76940082b62c85c3756f461cb2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 16:30:01 +0200 Subject: Clearing entities in the loop fail on Postgres It looks like when you clear entities on Postgres some references are lost and tags are not saved :-/ --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 9cd3dcb8..581ec178 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -172,15 +172,12 @@ abstract class WallabagImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); } ++$i; } $this->em->flush(); + $this->em->clear(); } /** -- cgit v1.2.3 From c98db1b653b5dc8b701422190b02d9fbf10c4e68 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 4 Sep 2016 21:49:21 +0200 Subject: Convert other imports to Rabbit --- .../ImportBundle/Import/WallabagImport.php | 118 ++++++++------------- 1 file changed, 43 insertions(+), 75 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 581ec178..6ad14e8c 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -11,7 +11,6 @@ abstract class WallabagImport extends AbstractImport protected $skippedEntries = 0; protected $importedEntries = 0; protected $filepath; - protected $markAsRead; // untitled in all languages from v1 protected $untitled = [ 'Untitled', @@ -28,19 +27,6 @@ abstract class WallabagImport extends AbstractImport '', ]; - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } - /** * {@inheritdoc} */ @@ -79,6 +65,12 @@ abstract class WallabagImport extends AbstractImport return false; } + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); return true; @@ -108,85 +100,61 @@ abstract class WallabagImport extends AbstractImport } /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead + * {@inheritdoc} */ - public function setMarkAsRead($markAsRead) + public function parseEntry(array $importedEntry) { - $this->markAsRead = $markAsRead; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); - return $this; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - /** - * Parse and insert all given entries. - * - * @param $entries - */ - protected function parseEntries($entries) - { - $i = 1; + return; + } + + $data = $this->prepareEntry($importedEntry); - foreach ($entries as $importedEntry) { - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + $entry = $this->fetchContent( + new Entry($this->user), + $importedEntry['url'], + $data + ); - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; - $data = $this->prepareEntry($importedEntry, $this->markAsRead); + return; + } - $entry = $this->fetchContent( - new Entry($this->user), - $importedEntry['url'], - $data + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] ); + } - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; - } - - if (array_key_exists('tags', $data)) { - $this->contentProxy->assignTagsToEntry( - $entry, - $data['tags'] - ); - } - - if (isset($importedEntry['preview_picture'])) { - $entry->setPreviewPicture($importedEntry['preview_picture']); - } - - $entry->setArchived($data['is_archived']); - $entry->setStarred($data['is_starred']); - - $this->em->persist($entry); - ++$this->importedEntries; - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } - ++$i; + if (isset($importedEntry['preview_picture'])) { + $entry->setPreviewPicture($importedEntry['preview_picture']); } - $this->em->flush(); - $this->em->clear(); + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; } /** * This should return a cleaned array for a given entry to be given to `updateEntry`. * - * @param array $entry Data from the imported file - * @param bool $markAsRead Should we mark as read content? + * @param array $entry Data from the imported file * * @return array */ - abstract protected function prepareEntry($entry = [], $markAsRead = false); + abstract protected function prepareEntry($entry = []); } -- cgit v1.2.3 From 3849a9f3231c0109c87af085452c3ac5e4aed303 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 07:50:10 +0200 Subject: Some cleanup & refactor --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 6ad14e8c..8e18e0ef 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -3,11 +3,9 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; abstract class WallabagImport extends AbstractImport { - protected $user; protected $skippedEntries = 0; protected $importedEntries = 0; protected $filepath; -- cgit v1.2.3 From 3aca0a9f00417b64203a660dee0a2b4c0fe22ac8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 09:35:42 +0200 Subject: CS --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 8e18e0ef..026567b0 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -6,8 +6,6 @@ use Wallabag\CoreBundle\Entity\Entry; abstract class WallabagImport extends AbstractImport { - protected $skippedEntries = 0; - protected $importedEntries = 0; protected $filepath; // untitled in all languages from v1 protected $untitled = [ -- cgit v1.2.3 From 6d65c0a8b089d3caa6f8e20d7935a9fe2f87d926 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 09:36:07 +0200 Subject: Add ability to define created_at for all import At the moment only Readability & wallabag v2 import allow created_at import. Pocket removed `time_added` field from their API v2 to v3... And wallabag v1 doesn't export that value. --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 026567b0..8e50b135 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -139,6 +139,10 @@ abstract class WallabagImport extends AbstractImport $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); + if (!empty($data['created_at'])) { + $entry->setCreatedAt(new \DateTime($data['created_at'])); + } + $this->em->persist($entry); ++$this->importedEntries; -- cgit v1.2.3 From c80cc01afa315dcfa38b2a01c5b05d4516659c24 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 21:09:05 +0200 Subject: Change flash message for queued articles --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 8e50b135..969a6a04 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -72,17 +72,6 @@ abstract class WallabagImport extends AbstractImport return true; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * Set file path to the json file. * -- cgit v1.2.3 From 59b97fae996d8307b9d957d210d46200f6d206bf Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 17 Sep 2016 07:40:56 +0200 Subject: Avoid losing entry when fetching fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of just say “Failed to save entry” we’ll save the entry at all cost and try to fetch content. If fetching content failed, the entry will still be saved at least, but without content. --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 969a6a04..043bb0a2 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -101,18 +101,12 @@ abstract class WallabagImport extends AbstractImport $data = $this->prepareEntry($importedEntry); - $entry = $this->fetchContent( - new Entry($this->user), - $importedEntry['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); if (array_key_exists('tags', $data)) { $this->contentProxy->assignTagsToEntry( -- cgit v1.2.3 From 401135852c6b25c8d5ab97beaefb02d1bd023ec9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 25 Sep 2016 11:26:15 +0200 Subject: Use scheduled entity insertions to avoid tag duplicate Using `getScheduledEntityInsertions()` we can retrieve not yet flushed but already persisted entities and then avoid tags duplication on import. --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 043bb0a2..3754e4a9 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -111,7 +111,8 @@ abstract class WallabagImport extends AbstractImport if (array_key_exists('tags', $data)) { $this->contentProxy->assignTagsToEntry( $entry, - $data['tags'] + $data['tags'], + $this->em->getUnitOfWork()->getScheduledEntityInsertions() ); } -- cgit v1.2.3 From c7ea9b41f32f222fef6a59734ea0b1176bfa1f41 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 27 Sep 2016 17:01:14 +0200 Subject: Add controller test for Instapaper --- src/Wallabag/ImportBundle/Import/WallabagImport.php | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/Wallabag/ImportBundle/Import/WallabagImport.php') diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 3754e4a9..702da057 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -58,6 +58,8 @@ abstract class WallabagImport extends AbstractImport $data = json_decode(file_get_contents($this->filepath), true); if (empty($data)) { + $this->logger->error('WallabagImport: no entries in imported file'); + return false; } -- cgit v1.2.3