From ff1a5362f7254d686864ea53994da6c517b3d3e8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 27 Sep 2016 07:57:53 +0200 Subject: Add Instapaper import Also update ImportController with latest import (chrome, firefox & instapaper). --- .../ImportBundle/Command/RedisWorkerCommand.php | 2 +- .../ImportBundle/Controller/ImportController.php | 12 +- .../Controller/InstapaperController.php | 77 ++++++++++++ .../ImportBundle/Import/AbstractImport.php | 4 + .../ImportBundle/Import/InstapaperImport.php | 134 +++++++++++++++++++++ .../ImportBundle/Resources/config/rabbit.yml | 7 ++ .../ImportBundle/Resources/config/redis.yml | 20 +++ .../ImportBundle/Resources/config/services.yml | 10 ++ .../Resources/views/Instapaper/index.html.twig | 45 +++++++ 9 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/InstapaperController.php create mode 100644 src/Wallabag/ImportBundle/Import/InstapaperImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig (limited to 'src') diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php index 5f90e00f..c2c11f11 100644 --- a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php +++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php @@ -17,7 +17,7 @@ class RedisWorkerCommand extends ContainerAwareCommand $this ->setName('wallabag:import:redis-worker') ->setDescription('Launch Redis worker') - ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability') + ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, firefox, chrome or instapaper') ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false) ; } diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 36a2a399..15de75ff 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -38,7 +38,11 @@ class ImportController extends Controller $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket') + $this->getTotalMessageInRabbitQueue('readability') + $this->getTotalMessageInRabbitQueue('wallabag_v1') - + $this->getTotalMessageInRabbitQueue('wallabag_v2'); + + $this->getTotalMessageInRabbitQueue('wallabag_v2') + + $this->getTotalMessageInRabbitQueue('firefox') + + $this->getTotalMessageInRabbitQueue('chrome') + + $this->getTotalMessageInRabbitQueue('instapaper') + ; } catch (\Exception $e) { $rabbitNotInstalled = true; } @@ -49,7 +53,11 @@ class ImportController extends Controller $nbRedisMessages = $redis->llen('wallabag.import.pocket') + $redis->llen('wallabag.import.readability') + $redis->llen('wallabag.import.wallabag_v1') - + $redis->llen('wallabag.import.wallabag_v2'); + + $redis->llen('wallabag.import.wallabag_v2') + + $redis->llen('wallabag.import.firefox') + + $redis->llen('wallabag.import.chrome') + + $redis->llen('wallabag.import.instapaper') + ; } catch (\Exception $e) { $redisNotInstalled = true; } diff --git a/src/Wallabag/ImportBundle/Controller/InstapaperController.php b/src/Wallabag/ImportBundle/Controller/InstapaperController.php new file mode 100644 index 00000000..c3fc8a39 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/InstapaperController.php @@ -0,0 +1,77 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $instapaper = $this->get('wallabag_import.instapaper.import'); + $instapaper->setUser($this->getUser()); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $instapaper->setProducer($this->get('old_sound_rabbit_mq.import_instapaper_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $instapaper->setProducer($this->get('wallabag_import.producer.redis.instapaper')); + } + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = 'instapaper_'.$this->getUser()->getId().'.csv'; + + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $instapaper + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $instapaper->getSummary(); + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render('WallabagImportBundle:Instapaper:index.html.twig', [ + 'form' => $form->createView(), + 'import' => $instapaper, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index a1a14576..764b390a 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -106,6 +106,10 @@ abstract class AbstractImport implements ImportInterface $i = 1; foreach ($entries as $importedEntry) { + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + $entry = $this->parseEntry($importedEntry); if (null === $entry) { diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php new file mode 100644 index 00000000..356acf23 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php @@ -0,0 +1,134 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('InstapaperImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('InstapaperImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $entries = []; + $handle = fopen($this->filepath, 'r'); + while (($data = fgetcsv($handle, 10240)) !== false) { + if ('URL' === $data[0]) { + continue; + } + + $entries[] = [ + 'url' => $data[0], + 'title' => $data[1], + 'status' => $data[3], + 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred', + 'is_starred' => $data[3] === 'Starred', + 'content_type' => '', + 'language' => '', + ]; + } + fclose($handle); + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $entry = new Entry($this->user); + $entry->setUrl($importedEntry['url']); + $entry->setTitle($importedEntry['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry); + + $entry->setArchived($importedEntry['is_archived']); + $entry->setStarred($importedEntry['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index 6ada6302..70b8a0d4 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -14,6 +14,13 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" - "@logger" + wallabag_import.consumer.amqp.instapaper: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.instapaper.import" + - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index c9c2cf26..0a81e1b5 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -20,6 +20,26 @@ services: - "@wallabag_import.readability.import" - "@logger" + # instapaper + wallabag_import.queue.redis.instapaper: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.instapaper" + + wallabag_import.producer.redis.instapaper: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.instapaper" + + wallabag_import.consumer.redis.instapaper: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.instapaper.import" + - "@logger" + # pocket wallabag_import.queue.redis.pocket: class: Simpleue\Queue\RedisQueue diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 990f336d..89adc71b 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -57,6 +57,16 @@ services: tags: - { name: wallabag_import.import, alias: readability } + wallabag_import.instapaper.import: + class: Wallabag\ImportBundle\Import\InstapaperImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: instapaper } + wallabag_import.firefox.import: class: Wallabag\ImportBundle\Import\FirefoxImport arguments: diff --git a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig new file mode 100644 index 00000000..5789361f --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig @@ -0,0 +1,45 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.instapaper.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + +
+
{{ import.description|trans }}
+

{{ 'import.instapaper.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} -- cgit v1.2.3