From 7816eb622df2353cea0ede0a3674d5eb3a01a1a9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:10:23 +0100 Subject: Add entry.saved event to import & rest --- .../ImportBundle/Import/AbstractImport.php | 24 +++++++++++++++++++++- src/Wallabag/ImportBundle/Import/BrowserImport.php | 17 +++++++++++++++ src/Wallabag/ImportBundle/Import/PocketImport.php | 7 ------- 3 files changed, 40 insertions(+), 8 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 764b390a..1d4a6e27 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractImport implements ImportInterface { protected $em; protected $logger; protected $contentProxy; + protected $eventDispatcher; protected $producer; protected $user; protected $markAsRead; @@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface protected $importedEntries = 0; protected $queuedEntries = 0; - public function __construct(EntityManager $em, ContentProxy $contentProxy) + public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher) { $this->em = $em; $this->logger = new NullLogger(); $this->contentProxy = $contentProxy; + $this->eventDispatcher = $eventDispatcher; } public function setLogger(LoggerInterface $logger) @@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ($this->markAsRead) { @@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface continue; } + // store each entry to be flushed so we can trigger the entry.saved event for each of them + // entry.saved needs the entry to be persisted in db because it needs it id to generate + // images (at least) + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); @@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 2ca1683b..8bf7d92e 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class BrowserImport extends AbstractImport { @@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ((array) $importedEntry !== $importedEntry) { @@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport continue; } + // @see AbstractImport + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; } ++$i; } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 327e2500..0c26aced 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -16,13 +16,6 @@ class PocketImport extends AbstractImport const NB_ELEMENTS = 5000; - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->contentProxy = $contentProxy; - $this->logger = new NullLogger(); - } - /** * Only used for test purpose. * -- cgit v1.2.3 From e46fdf3096d55028611713b7a196401aa57d864b Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:26:14 +0100 Subject: CS --- src/Wallabag/ImportBundle/Import/PocketImport.php | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 0c26aced..33093480 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,8 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; -- cgit v1.2.3 From 9ab024b4f5388e2a41c50a8a2b79e4033788782a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 4 Nov 2016 22:44:31 +0100 Subject: Add Pinboard import --- .../ImportBundle/Import/PinboardImport.php | 143 +++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 src/Wallabag/ImportBundle/Import/PinboardImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php new file mode 100644 index 00000000..9bcfbc36 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php @@ -0,0 +1,143 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('PinboardImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('PinboardImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + $this->logger->error('PinboardImport: no entries in imported file'); + + return false; + } + + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + + $this->parseEntries($data); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['href'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $data = [ + 'title' => $importedEntry['description'], + 'url' => $importedEntry['href'], + 'content_type' => '', + 'language' => '', + 'is_archived' => ('no' === $importedEntry['toread']) || $this->markAsRead, + 'is_starred' => false, + 'created_at' => $importedEntry['time'], + 'tags' => explode(' ', $importedEntry['tags']), + ]; + + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); + + if (!empty($data['tags'])) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'], + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } + + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + $entry->setCreatedAt(new \DateTime($data['created_at'])); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['toread'] = 'no'; + + return $importedEntry; + } +} -- cgit v1.2.3 From 36e6ef52a176ef654eade931a23f60fd91344f2f Mon Sep 17 00:00:00 2001 From: Jerome Charaoui Date: Fri, 2 Dec 2016 22:42:36 -0500 Subject: Imported entries which fail to fetch get standard error body --- src/Wallabag/ImportBundle/Import/ChromeImport.php | 2 +- src/Wallabag/ImportBundle/Import/FirefoxImport.php | 2 +- src/Wallabag/ImportBundle/Import/InstapaperImport.php | 1 + src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index d7620bcb..1a324934 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -37,7 +37,7 @@ class ChromeImport extends BrowserImport { $data = [ 'title' => $entry['name'], - 'html' => '', + 'html' => false, 'url' => $entry['url'], 'is_archived' => $this->markAsRead, 'tags' => '', diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index e010f5a4..d3f99770 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -37,7 +37,7 @@ class FirefoxImport extends BrowserImport { $data = [ 'title' => $entry['title'], - 'html' => '', + 'html' => false, 'url' => $entry['uri'], 'is_archived' => $this->markAsRead, 'tags' => '', diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php index cf4c785c..146a8c7c 100644 --- a/src/Wallabag/ImportBundle/Import/InstapaperImport.php +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php @@ -76,6 +76,7 @@ class InstapaperImport extends AbstractImport 'is_starred' => $data[3] === 'Starred', 'content_type' => '', 'language' => '', + 'html' => false, ]; } fclose($handle); diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index b8c0f777..66b00885 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -103,6 +103,7 @@ class ReadabilityImport extends AbstractImport 'is_archived' => $importedEntry['archive'] || $this->markAsRead, 'is_starred' => $importedEntry['favorite'], 'created_at' => $importedEntry['date_added'], + 'html' => false, ]; $entry = new Entry($this->user); -- cgit v1.2.3 From 3cd6da0b749bd136758add109ed30531be224f4a Mon Sep 17 00:00:00 2001 From: Jerome Charaoui Date: Fri, 2 Dec 2016 23:55:30 -0500 Subject: Allow fetching content on all import types For better consistency, allow all types of imported entries to update, which was already the case for Firefox and Chrome. --- src/Wallabag/ImportBundle/Import/InstapaperImport.php | 2 -- src/Wallabag/ImportBundle/Import/PinboardImport.php | 2 -- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 2 -- src/Wallabag/ImportBundle/Import/WallabagV1Import.php | 2 -- 4 files changed, 8 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php index 146a8c7c..70a53f1a 100644 --- a/src/Wallabag/ImportBundle/Import/InstapaperImport.php +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php @@ -74,8 +74,6 @@ class InstapaperImport extends AbstractImport 'status' => $data[3], 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred', 'is_starred' => $data[3] === 'Starred', - 'content_type' => '', - 'language' => '', 'html' => false, ]; } diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php index 9bcfbc36..d9865534 100644 --- a/src/Wallabag/ImportBundle/Import/PinboardImport.php +++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php @@ -98,8 +98,6 @@ class PinboardImport extends AbstractImport $data = [ 'title' => $importedEntry['description'], 'url' => $importedEntry['href'], - 'content_type' => '', - 'language' => '', 'is_archived' => ('no' === $importedEntry['toread']) || $this->markAsRead, 'is_starred' => false, 'created_at' => $importedEntry['time'], diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 66b00885..de320d23 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -98,8 +98,6 @@ class ReadabilityImport extends AbstractImport $data = [ 'title' => $importedEntry['article__title'], 'url' => $importedEntry['article__url'], - 'content_type' => '', - 'language' => '', 'is_archived' => $importedEntry['archive'] || $this->markAsRead, 'is_starred' => $importedEntry['favorite'], 'created_at' => $importedEntry['date_added'], diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 4f001062..59e3ce02 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -37,8 +37,6 @@ class WallabagV1Import extends WallabagImport 'title' => $entry['title'], 'html' => $entry['content'], 'url' => $entry['url'], - 'content_type' => '', - 'language' => '', 'is_archived' => $entry['is_read'] || $this->markAsRead, 'is_starred' => $entry['is_fav'], 'tags' => '', -- cgit v1.2.3 From bb98fede2b7c3e6eba8d47b2ce560d824d166b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 7 Dec 2016 16:01:50 +0100 Subject: Fixed imports with is_starred and is_archived --- src/Wallabag/ImportBundle/Import/ChromeImport.php | 3 ++- src/Wallabag/ImportBundle/Import/FirefoxImport.php | 3 ++- src/Wallabag/ImportBundle/Import/WallabagV2Import.php | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 1a324934..2667890f 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -39,7 +39,8 @@ class ChromeImport extends BrowserImport 'title' => $entry['name'], 'html' => false, 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, + 'is_archived' => (int) $this->markAsRead, + 'is_starred' => false, 'tags' => '', 'created_at' => substr($entry['date_added'], 0, 10), ]; diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index d3f99770..c50c69b3 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -39,7 +39,8 @@ class FirefoxImport extends BrowserImport 'title' => $entry['title'], 'html' => false, 'url' => $entry['uri'], - 'is_archived' => $this->markAsRead, + 'is_archived' => (int) $this->markAsRead, + 'is_starred' => false, 'tags' => '', 'created_at' => substr($entry['dateAdded'], 0, 10), ]; diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php index 37c8ca14..d2a89d79 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php @@ -36,7 +36,8 @@ class WallabagV2Import extends WallabagImport return [ 'html' => $entry['content'], 'content_type' => $entry['mimetype'], - 'is_archived' => ($entry['is_archived'] || $this->markAsRead), + 'is_archived' => (int) ($entry['is_archived'] || $this->markAsRead), + 'is_starred' => false, ] + $entry; } -- cgit v1.2.3