From ef75e1220ebb76a8df019d946460ad612759f0bb Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 3 Sep 2016 17:36:57 +0200 Subject: Send every imported item to the queue Instead of queing real Entry to process, we queue all the item to import from Pocket in a raw format. Then, the worker retrieve that information, find / create the entry and save it. --- .../ImportBundle/Controller/PocketController.php | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 36ee25bf..a2dcd8a7 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -10,12 +10,29 @@ use Symfony\Component\Form\Extension\Core\Type\CheckboxType; class PocketController extends Controller { + /** + * Return Pocket Import Service with or without RabbitMQ enabled. + * + * @return \Wallabag\ImportBundle\Import\PocketImport + */ + private function getPocketImportService() + { + $pocket = $this->get('wallabag_import.pocket.import'); + $pocket->setUser($this->getUser()); + + if ($this->get('craue_config')->get('rabbitmq')) { + $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.wallabag_pocket_producer')); + } + + return $pocket; + } + /** * @Route("/pocket", name="import_pocket") */ public function indexAction() { - $pocket = $this->get('wallabag_import.pocket.import'); + $pocket = $this->getPocketImportService(); $form = $this->createFormBuilder($pocket) ->add('mark_as_read', CheckboxType::class, [ 'label' => 'import.form.mark_as_read_label', @@ -24,7 +41,7 @@ class PocketController extends Controller ->getForm(); return $this->render('WallabagImportBundle:Pocket:index.html.twig', [ - 'import' => $this->get('wallabag_import.pocket.import'), + 'import' => $this->getPocketImportService(), 'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true, 'form' => $form->createView(), ]); @@ -35,7 +52,7 @@ class PocketController extends Controller */ public function authAction(Request $request) { - $requestToken = $this->get('wallabag_import.pocket.import') + $requestToken = $this->getPocketImportService() ->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL)); if (false === $requestToken) { @@ -62,7 +79,7 @@ class PocketController extends Controller public function callbackAction() { $message = 'flashes.import.notice.failed'; - $pocket = $this->get('wallabag_import.pocket.import'); + $pocket = $this->getPocketImportService(); $markAsRead = $this->get('session')->get('mark_as_read'); $this->get('session')->remove('mark_as_read'); -- cgit v1.2.3 From c98db1b653b5dc8b701422190b02d9fbf10c4e68 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 4 Sep 2016 21:49:21 +0200 Subject: Convert other imports to Rabbit --- src/Wallabag/ImportBundle/Controller/PocketController.php | 2 +- src/Wallabag/ImportBundle/Controller/ReadabilityController.php | 6 +++++- src/Wallabag/ImportBundle/Controller/WallabagController.php | 2 +- src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php | 8 +++++++- src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php | 8 +++++++- 5 files changed, 21 insertions(+), 5 deletions(-) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index a2dcd8a7..57c007c3 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -21,7 +21,7 @@ class PocketController extends Controller $pocket->setUser($this->getUser()); if ($this->get('craue_config')->get('rabbitmq')) { - $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.wallabag_pocket_producer')); + $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); } return $pocket; diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index b61aa99c..ee875a40 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -18,6 +18,11 @@ class ReadabilityController extends Controller $form->handleRequest($request); $readability = $this->get('wallabag_import.readability.import'); + $readability->setUser($this->getUser()); + + if ($this->get('craue_config')->get('rabbitmq')) { + $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); + } if ($form->isValid()) { $file = $form->get('file')->getData(); @@ -26,7 +31,6 @@ class ReadabilityController extends Controller if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $readability - ->setUser($this->getUser()) ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) ->import(); diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php index 76ced0d2..1e6114c5 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagController.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php @@ -38,6 +38,7 @@ abstract class WallabagController extends Controller $form->handleRequest($request); $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); if ($form->isValid()) { $file = $form->get('file')->getData(); @@ -46,7 +47,6 @@ abstract class WallabagController extends Controller if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $wallabag - ->setUser($this->getUser()) ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) ->import(); diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php index 3e748d57..f80aec3a 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php @@ -12,7 +12,13 @@ class WallabagV1Controller extends WallabagController */ protected function getImportService() { - return $this->get('wallabag_import.wallabag_v1.import'); + $service = $this->get('wallabag_import.wallabag_v1.import'); + + if ($this->get('craue_config')->get('rabbitmq')) { + $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } + + return $service; } /** diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php index c2a42165..063cddd9 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php @@ -12,7 +12,13 @@ class WallabagV2Controller extends WallabagController */ protected function getImportService() { - return $this->get('wallabag_import.wallabag_v2.import'); + $service = $this->get('wallabag_import.wallabag_v2.import'); + + if ($this->get('craue_config')->get('rabbitmq')) { + $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); + } + + return $service; } /** -- cgit v1.2.3 From b3437d58ae224121375c99e9288d8b808524e624 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 21:02:03 +0200 Subject: Enable Redis async import - using javibravo/simpleue - internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear - if both option are enable rabbit will be choosen - services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml` - --- src/Wallabag/ImportBundle/Controller/PocketController.php | 6 ++++-- src/Wallabag/ImportBundle/Controller/ReadabilityController.php | 6 ++++-- src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php | 6 ++++-- src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 57c007c3..3d555717 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -20,8 +20,10 @@ class PocketController extends Controller $pocket = $this->get('wallabag_import.pocket.import'); $pocket->setUser($this->getUser()); - if ($this->get('craue_config')->get('rabbitmq')) { - $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket')); } return $pocket; diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index ee875a40..61243042 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -20,8 +20,10 @@ class ReadabilityController extends Controller $readability = $this->get('wallabag_import.readability.import'); $readability->setUser($this->getUser()); - if ($this->get('craue_config')->get('rabbitmq')) { - $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $readability->setProducer($this->get('wallabag_import.producer.redis.readability')); } if ($form->isValid()) { diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php index f80aec3a..312c7a35 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php @@ -14,8 +14,10 @@ class WallabagV1Controller extends WallabagController { $service = $this->get('wallabag_import.wallabag_v1.import'); - if ($this->get('craue_config')->get('rabbitmq')) { - $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1')); } return $service; diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php index 063cddd9..45211fe6 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php @@ -14,8 +14,10 @@ class WallabagV2Controller extends WallabagController { $service = $this->get('wallabag_import.wallabag_v2.import'); - if ($this->get('craue_config')->get('rabbitmq')) { - $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2')); } return $service; -- cgit v1.2.3 From 015c7a8359c950f9621b38b11c3973860a981da8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 20:24:04 +0200 Subject: Add more tests And ability to define how many messages can be hanle by the redis worker before stopping (usefull for tests) --- src/Wallabag/ImportBundle/Controller/ReadabilityController.php | 2 +- src/Wallabag/ImportBundle/Controller/WallabagController.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index 61243042..8775e8a3 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -31,7 +31,7 @@ class ReadabilityController extends Controller $markAsRead = $form->get('mark_as_read')->getData(); $name = 'readability_'.$this->getUser()->getId().'.json'; - if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $readability ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php index 1e6114c5..3c2dd6d1 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagController.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php @@ -45,7 +45,7 @@ abstract class WallabagController extends Controller $markAsRead = $form->get('mark_as_read')->getData(); $name = $this->getUser()->getId().'.json'; - if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $wallabag ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) -- cgit v1.2.3 From e01a3c98d6908e95121b5ade0161f40af1b05ca6 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 20:31:32 +0200 Subject: Display how many messages are queue - update the docker-composer to add Redis - add migrations --- .../ImportBundle/Controller/ImportController.php | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index c1486e38..d670746c 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -16,4 +16,58 @@ class ImportController extends Controller 'imports' => $this->get('wallabag_import.chain')->getAll(), ]); } + + /** + * Display how many messages are queue (both in Redis and RabbitMQ). + */ + public function checkQueueAction() + { + $nbRedisMessages = null; + $nbRabbitMessages = null; + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket') + + $this->getTotalMessageInRabbitQueue('readability') + + $this->getTotalMessageInRabbitQueue('wallabag_v1') + + $this->getTotalMessageInRabbitQueue('wallabag_v2') + ; + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $redis = $this->get('wallabag_core.redis.client'); + + $nbRedisMessages = $redis->llen('wallabag.import.pocket') + + $redis->llen('wallabag.import.readability') + + $redis->llen('wallabag.import.wallabag_v1') + + $redis->llen('wallabag.import.wallabag_v2') + ; + } + + return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [ + 'nbRedisMessages' => $nbRedisMessages, + 'nbRabbitMessages' => $nbRabbitMessages, + ]); + } + + /** + * Count message in RabbitMQ queue. + * It get one message without acking it (so it'll stay in the queue) + * which will include the total of *other* messages in the queue. + * Adding one to that messages will result in the full total message. + * + * @param string $importService The import service related: pocket, readability, wallabag_v1 or wallabag_v2 + * + * @return int + */ + private function getTotalMessageInRabbitQueue($importService) + { + $message = $this + ->get('old_sound_rabbit_mq.import_'.$importService.'_consumer') + ->getChannel() + ->basic_get('wallabag.import.'.$importService); + + if (null === $message) { + return 0; + } + + return $message->delivery_info['message_count'] + 1; + } } -- cgit v1.2.3 From c80cc01afa315dcfa38b2a01c5b05d4516659c24 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 21:09:05 +0200 Subject: Change flash message for queued articles --- src/Wallabag/ImportBundle/Controller/PocketController.php | 6 ++++++ src/Wallabag/ImportBundle/Controller/ReadabilityController.php | 6 ++++++ src/Wallabag/ImportBundle/Controller/WallabagController.php | 6 ++++++ 3 files changed, 18 insertions(+) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 3d555717..1f92c182 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -102,6 +102,12 @@ class PocketController extends Controller '%imported%' => $summary['imported'], '%skipped%' => $summary['skipped'], ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } } $this->get('session')->getFlashBag()->add( diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index 8775e8a3..d00e22c2 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -46,6 +46,12 @@ class ReadabilityController extends Controller '%skipped%' => $summary['skipped'], ]); + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); } diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php index 3c2dd6d1..9c0cde80 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagController.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php @@ -60,6 +60,12 @@ abstract class WallabagController extends Controller '%skipped%' => $summary['skipped'], ]); + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); } -- cgit v1.2.3 From 4fc998245c56ad95c1e753ab52b0c702d4a8a59d Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 20:08:07 +0200 Subject: Only display message in queue for admin Instead of for EVERYONE --- src/Wallabag/ImportBundle/Controller/ImportController.php | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index d670746c..ea4f7d7b 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -19,12 +19,20 @@ class ImportController extends Controller /** * Display how many messages are queue (both in Redis and RabbitMQ). + * Only for admins. */ public function checkQueueAction() { $nbRedisMessages = null; $nbRabbitMessages = null; + if (!$this->get('security.authorization_checker')->isGranted('ROLE_SUPER_ADMIN')) { + return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [ + 'nbRedisMessages' => $nbRedisMessages, + 'nbRabbitMessages' => $nbRabbitMessages, + ]); + } + if ($this->get('craue_config')->get('import_with_rabbitmq')) { $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket') + $this->getTotalMessageInRabbitQueue('readability') -- cgit v1.2.3 From ebe0787e093f4f2934430033015d6ebad1c64dca Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 22:22:25 +0200 Subject: Moved Pocket token to user config --- src/Wallabag/ImportBundle/Controller/PocketController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Wallabag/ImportBundle/Controller') diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 1f92c182..56be5cbf 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -44,7 +44,7 @@ class PocketController extends Controller return $this->render('WallabagImportBundle:Pocket:index.html.twig', [ 'import' => $this->getPocketImportService(), - 'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true, + 'has_consumer_key' => '' === trim($this->getUser()->getConfig()->getPocketConsumerKey()) ? false : true, 'form' => $form->createView(), ]); } -- cgit v1.2.3