From 59201088b4fc13fd361238396f630dabd9bd1990 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 17:47:47 +0200 Subject: bring chrome and firefox as separate imports --- .../ImportBundle/Command/ImportCommand.php | 9 +- .../ImportBundle/Controller/BrowserController.php | 23 +- .../ImportBundle/Controller/ChromeController.php | 41 ++++ .../ImportBundle/Controller/FirefoxController.php | 41 ++++ src/Wallabag/ImportBundle/Import/BrowserImport.php | 247 ++++++++++----------- src/Wallabag/ImportBundle/Import/ChromeImport.php | 71 ++++++ src/Wallabag/ImportBundle/Import/FirefoxImport.php | 71 ++++++ .../ImportBundle/Resources/config/rabbit.yml | 14 ++ .../ImportBundle/Resources/config/redis.yml | 40 ++++ .../ImportBundle/Resources/config/services.yml | 15 +- .../Resources/views/Browser/index.html.twig | 43 ---- .../Resources/views/Chrome/index.html.twig | 43 ++++ .../Resources/views/Firefox/index.html.twig | 43 ++++ 13 files changed, 513 insertions(+), 188 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/ChromeController.php create mode 100644 src/Wallabag/ImportBundle/Controller/FirefoxController.php create mode 100644 src/Wallabag/ImportBundle/Import/ChromeImport.php create mode 100644 src/Wallabag/ImportBundle/Import/FirefoxImport.php delete mode 100644 src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig create mode 100644 src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig create mode 100644 src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig (limited to 'src/Wallabag/ImportBundle') diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index ac3d1d92..1df38295 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -17,7 +17,7 @@ class ImportCommand extends ContainerAwareCommand ->setDescription('Import entries from a JSON export from a wallabag v1 instance') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2 or browser', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -44,8 +44,11 @@ class ImportCommand extends ContainerAwareCommand case 'v2': $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); break; - case 'browser': - $wallabag = $this->getContainer()->get('wallabag_import.browser.import'); + case 'firefox': + $wallabag = $this->getContainer()->get('wallabag_import.firefox.import'); + break; + case 'chrome': + $wallabag = $this->getContainer()->get('wallabag_import.chrome.import'); break; case 'v1': default: diff --git a/src/Wallabag/ImportBundle/Controller/BrowserController.php b/src/Wallabag/ImportBundle/Controller/BrowserController.php index 3b54a72e..144a4880 100644 --- a/src/Wallabag/ImportBundle/Controller/BrowserController.php +++ b/src/Wallabag/ImportBundle/Controller/BrowserController.php @@ -8,27 +8,21 @@ use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Wallabag\ImportBundle\Form\Type\UploadImportType; -class BrowserController extends Controller +abstract class BrowserController extends Controller { /** * Return the service to handle the import. * * @return \Wallabag\ImportBundle\Import\ImportInterface */ - protected function getImportService() - { - return $this->get('wallabag_import.browser.import'); - } + abstract protected function getImportService(); /** * Return the template used for the form. * * @return string */ - protected function getImportTemplate() - { - return 'WallabagImportBundle:Browser:index.html.twig'; - } + abstract protected function getImportTemplate(); /** * @Route("/browser", name="import_browser") @@ -43,15 +37,15 @@ class BrowserController extends Controller $form->handleRequest($request); $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); if ($form->isValid()) { $file = $form->get('file')->getData(); $markAsRead = $form->get('mark_as_read')->getData(); $name = $this->getUser()->getId().'.json'; - if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $wallabag - ->setUser($this->getUser()) ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) ->import(); @@ -60,12 +54,17 @@ class BrowserController extends Controller if (true === $res) { $summary = $wallabag->getSummary(); - // TODO : Pluralize these messages $message = $this->get('translator')->trans('flashes.import.notice.summary', [ '%imported%' => $summary['imported'], '%skipped%' => $summary['skipped'], ]); + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); } diff --git a/src/Wallabag/ImportBundle/Controller/ChromeController.php b/src/Wallabag/ImportBundle/Controller/ChromeController.php new file mode 100644 index 00000000..e4cc322a --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/ChromeController.php @@ -0,0 +1,41 @@ +get('wallabag_import.chrome.import'); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1')); + } + + return $service; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Chrome:index.html.twig'; + } + + /** + * @Route("/chrome", name="import_chrome") + */ + public function indexAction(Request $request) + { + return parent::indexAction($request); + } +} diff --git a/src/Wallabag/ImportBundle/Controller/FirefoxController.php b/src/Wallabag/ImportBundle/Controller/FirefoxController.php new file mode 100644 index 00000000..e0dd8214 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/FirefoxController.php @@ -0,0 +1,41 @@ +get('wallabag_import.firefox.import'); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1')); + } + + return $service; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Firefox:index.html.twig'; + } + + /** + * @Route("/firefox", name="import_firefox") + */ + public function indexAction(Request $request) + { + return parent::indexAction($request); + } +} diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index e3457196..ef7d6d95 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -9,69 +9,24 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; -class BrowserImport implements ImportInterface +abstract class BrowserImport extends AbstractImport { - protected $user; - protected $em; - protected $logger; - protected $contentProxy; - protected $skippedEntries = 0; - protected $importedEntries = 0; - protected $totalEntries = 0; protected $filepath; - protected $markAsRead; - private $nbEntries; - - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->logger = new NullLogger(); - $this->contentProxy = $contentProxy; - } - - public function setLogger(LoggerInterface $logger) - { - $this->logger = $logger; - } - - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - * - * @return $this - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } /** * {@inheritdoc} */ - public function getName() - { - return 'Firefox & Google Chrome'; - } + abstract public function getName(); /** * {@inheritdoc} */ - public function getUrl() - { - return 'import_browser'; - } + abstract public function getUrl(); /** * {@inheritdoc} */ - public function getDescription() - { - return 'import.browser.description'; - } + abstract public function getDescription(); /** * {@inheritdoc} @@ -96,124 +51,162 @@ class BrowserImport implements ImportInterface return false; } - $this->nbEntries = 1; + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); - $this->em->flush(); return true; } - private function parseEntries($data) + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) { - foreach ($data as $importedEntry) { - $this->parseEntry($importedEntry); + $i = 1; + + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } + ++$i; } - $this->totalEntries += count($data); + + $this->em->flush(); } - private function parseEntry($importedEntry) + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) { - if (!is_array($importedEntry)) { - return; + foreach ($entries as $importedEntry) { + + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); } + } - /* Firefox uses guid while Chrome uses id */ + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); - return; } + if (key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); - return; } - if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { - - /* Firefox uses uri while Chrome uses url */ - $firefox = key_exists('uri', $importedEntry); - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; + if (!key_exists('uri', $importedEntry) && !key_exists('url', $importedEntry)) { + return; + } - return; - } + $firefox = key_exists('uri', $importedEntry); - if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { - $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); - ++$this->skippedEntries; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - return; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - try { - $entry = $this->contentProxy->updateEntry( - new Entry($this->user), - ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] - ); - } catch (\Exception $e) { - $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); - ++$this->skippedEntries; + return; + } - return; - } + $data = $this->prepareEntry($importedEntry); - $entry->setArchived($this->markAsRead); + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - $this->em->persist($entry); - ++$this->importedEntries; + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); - // flush every 20 entries - if (($this->nbEntries % 20) === 0) { - $this->em->flush(); - $this->em->clear($entry); - } - ++$this->nbEntries; + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] + ); } - } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - * - * @return $this - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; + $entry->setArchived($data['is_archived']); - return $this; - } + if (!empty($data['created_at'])) { + $dt = new \DateTime(); + $entry->setCreatedAt($dt->setTimestamp($data['created_at']/1000)); + } - /** - * Set file path to the json file. - * - * @param string $filepath - * - * @return $this - */ - public function setFilepath($filepath) - { - $this->filepath = $filepath; + $this->em->persist($entry); + ++$this->importedEntries; - return $this; + return $entry; } /** * {@inheritdoc} */ - public function getSummary() + protected function setEntryAsRead(array $importedEntry) { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; + $importedEntry['is_archived'] = 1; + + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php new file mode 100644 index 00000000..7936ee2f --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php new file mode 100644 index 00000000..cbf10b87 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index aa049749..6ada6302 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -28,3 +28,17 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" - "@logger" + wallabag_import.consumer.amqp.firefox: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.firefox.import" + - "@logger" + wallabag_import.consumer.amqp.chrome: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.chrome.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 7d3248e5..c47778b8 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -79,3 +79,43 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" - "@logger" + + # firefox + wallabag_import.queue.redis.firefox: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.firefox" + + wallabag_import.producer.redis.firefox: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.firefox" + + wallabag_import.consumer.redis.firefox: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.firefox.import" + - "@logger" + + # chrome + wallabag_import.queue.redis.chrome: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.chrome" + + wallabag_import.producer.redis.firefox: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.chrome" + + wallabag_import.consumer.redis.firefox: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.chrome.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index d8be5c28..990f336d 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -57,12 +57,21 @@ services: tags: - { name: wallabag_import.import, alias: readability } - wallabag_import.browser.import: - class: Wallabag\ImportBundle\Import\BrowserImport + wallabag_import.firefox.import: + class: Wallabag\ImportBundle\Import\FirefoxImport arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" calls: - [ setLogger, [ "@logger" ]] tags: - - { name: wallabag_import.import, alias: browser } + - { name: wallabag_import.import, alias: firefox } + wallabag_import.chrome.import: + class: Wallabag\ImportBundle\Import\ChromeImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: chrome } diff --git a/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig deleted file mode 100644 index bfc74e9d..00000000 --- a/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "WallabagCoreBundle::layout.html.twig" %} - -{% block title %}{{ 'import.browser.page_title'|trans }}{% endblock %} - -{% block content %} -
-
-
-
-
{{ import.description|trans|raw }}
-

{{ 'import.browser.how_to'|trans }}

- -
- {{ form_start(form, {'method': 'POST'}) }} - {{ form_errors(form) }} -
-
- {{ form_errors(form.file) }} -
- {{ form.file.vars.label|trans }} - {{ form_widget(form.file) }} -
-
- -
-
-
-
{{ 'import.form.mark_as_read_title'|trans }}
- {{ form_widget(form.mark_as_read) }} - {{ form_label(form.mark_as_read) }} -
-
- - {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} - - {{ form_rest(form) }} - -
-
-
-
-
-{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig new file mode 100644 index 00000000..ead828c6 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.chrome.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans|raw }}
+

{{ 'import.chrome.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig new file mode 100644 index 00000000..f975da3f --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.firefox.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans|raw }}
+

{{ 'import.firefox.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} -- cgit v1.2.3