From 9ab024b4f5388e2a41c50a8a2b79e4033788782a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 4 Nov 2016 22:44:31 +0100 Subject: Add Pinboard import --- .../ImportBundle/Command/ImportCommand.php | 5 +- .../ImportBundle/Command/RedisWorkerCommand.php | 2 +- .../ImportBundle/Controller/ImportController.php | 2 + .../ImportBundle/Controller/PinboardController.php | 77 +++++++++++ .../ImportBundle/Import/PinboardImport.php | 143 +++++++++++++++++++++ .../ImportBundle/Resources/config/rabbit.yml | 8 ++ .../ImportBundle/Resources/config/redis.yml | 21 +++ .../ImportBundle/Resources/config/services.yml | 11 ++ .../Resources/views/Pinboard/index.html.twig | 45 +++++++ 9 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/PinboardController.php create mode 100644 src/Wallabag/ImportBundle/Import/PinboardImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig (limited to 'src/Wallabag/ImportBundle') diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index e423ffae..28d01715 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -17,7 +17,7 @@ class ImportCommand extends ContainerAwareCommand ->setDescription('Import entries from a JSON export') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, readability, firefox or chrome', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, readability, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -56,6 +56,9 @@ class ImportCommand extends ContainerAwareCommand case 'instapaper': $import = $this->getContainer()->get('wallabag_import.instapaper.import'); break; + case 'pinboard': + $import = $this->getContainer()->get('wallabag_import.pinboard.import'); + break; default: $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); } diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php index c2c11f11..f793a314 100644 --- a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php +++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php @@ -17,7 +17,7 @@ class RedisWorkerCommand extends ContainerAwareCommand $this ->setName('wallabag:import:redis-worker') ->setDescription('Launch Redis worker') - ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, firefox, chrome or instapaper') + ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, pinboard, firefox, chrome or instapaper') ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false) ; } diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 15de75ff..237c748e 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -42,6 +42,7 @@ class ImportController extends Controller + $this->getTotalMessageInRabbitQueue('firefox') + $this->getTotalMessageInRabbitQueue('chrome') + $this->getTotalMessageInRabbitQueue('instapaper') + + $this->getTotalMessageInRabbitQueue('pinboard') ; } catch (\Exception $e) { $rabbitNotInstalled = true; @@ -57,6 +58,7 @@ class ImportController extends Controller + $redis->llen('wallabag.import.firefox') + $redis->llen('wallabag.import.chrome') + $redis->llen('wallabag.import.instapaper') + + $redis->llen('wallabag.import.pinboard') ; } catch (\Exception $e) { $redisNotInstalled = true; diff --git a/src/Wallabag/ImportBundle/Controller/PinboardController.php b/src/Wallabag/ImportBundle/Controller/PinboardController.php new file mode 100644 index 00000000..9c3f98d6 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/PinboardController.php @@ -0,0 +1,77 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $pinboard = $this->get('wallabag_import.pinboard.import'); + $pinboard->setUser($this->getUser()); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $pinboard->setProducer($this->get('old_sound_rabbit_mq.import_pinboard_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $pinboard->setProducer($this->get('wallabag_import.producer.redis.pinboard')); + } + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = 'pinboard_'.$this->getUser()->getId().'.json'; + + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $pinboard + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $pinboard->getSummary(); + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render('WallabagImportBundle:Pinboard:index.html.twig', [ + 'form' => $form->createView(), + 'import' => $pinboard, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php new file mode 100644 index 00000000..9bcfbc36 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php @@ -0,0 +1,143 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('PinboardImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('PinboardImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + $this->logger->error('PinboardImport: no entries in imported file'); + + return false; + } + + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + + $this->parseEntries($data); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['href'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $data = [ + 'title' => $importedEntry['description'], + 'url' => $importedEntry['href'], + 'content_type' => '', + 'language' => '', + 'is_archived' => ('no' === $importedEntry['toread']) || $this->markAsRead, + 'is_starred' => false, + 'created_at' => $importedEntry['time'], + 'tags' => explode(' ', $importedEntry['tags']), + ]; + + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); + + if (!empty($data['tags'])) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'], + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } + + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + $entry->setCreatedAt(new \DateTime($data['created_at'])); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['toread'] = 'no'; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index a5af5282..e9ecb846 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -24,6 +24,14 @@ services: - "@wallabag_import.instapaper.import" - "@event_dispatcher" - "@logger" + wallabag_import.consumer.amqp.pinboard: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.pinboard.import" + - "@event_dispatcher" + - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 5ced4c83..091cdba0 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -42,6 +42,27 @@ services: - "@event_dispatcher" - "@logger" + # pinboard + wallabag_import.queue.redis.pinboard: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.pinboard" + + wallabag_import.producer.redis.pinboard: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.pinboard" + + wallabag_import.consumer.redis.pinboard: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.pinboard.import" + - "@event_dispatcher" + - "@logger" + # pocket wallabag_import.queue.redis.pocket: class: Simpleue\Queue\RedisQueue diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 64822963..c4fe3f92 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -71,6 +71,17 @@ services: tags: - { name: wallabag_import.import, alias: instapaper } + wallabag_import.pinboard.import: + class: Wallabag\ImportBundle\Import\PinboardImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + - "@event_dispatcher" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: pinboard } + wallabag_import.firefox.import: class: Wallabag\ImportBundle\Import\FirefoxImport arguments: diff --git a/src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig new file mode 100644 index 00000000..43f196ad --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig @@ -0,0 +1,45 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.pinboard.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} + +
+
{{ import.description|trans }}
+

{{ 'import.pinboard.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} -- cgit v1.2.3