aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorJeremy Benoist <jeremy.benoist@gmail.com>2016-09-09 21:02:03 +0200
committerJeremy Benoist <jeremy.benoist@gmail.com>2016-09-11 21:58:56 +0200
commitb3437d58ae224121375c99e9288d8b808524e624 (patch)
tree94ce3446aed4396ba9304b8c97e421eba35e4edf
parent7f7531171f6e49110b5842f869e37c766a682473 (diff)
downloadwallabag-b3437d58ae224121375c99e9288d8b808524e624.tar.gz
wallabag-b3437d58ae224121375c99e9288d8b808524e624.tar.zst
wallabag-b3437d58ae224121375c99e9288d8b808524e624.zip
Enable Redis async import
- using javibravo/simpleue - internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear - if both option are enable rabbit will be choosen - services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml` -
-rw-r--r--app/config/config.yml8
-rw-r--r--app/config/parameters.yml.dist4
-rw-r--r--composer.json7
-rw-r--r--src/Wallabag/CoreBundle/Command/InstallCommand.php7
-rw-r--r--src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php7
-rw-r--r--src/Wallabag/CoreBundle/Resources/config/services.yml8
-rw-r--r--src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php41
-rw-r--r--src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php (renamed from src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php)6
-rw-r--r--src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php84
-rw-r--r--src/Wallabag/ImportBundle/Controller/PocketController.php6
-rw-r--r--src/Wallabag/ImportBundle/Controller/ReadabilityController.php6
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php6
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php6
-rw-r--r--src/Wallabag/ImportBundle/Import/AbstractImport.php8
-rw-r--r--src/Wallabag/ImportBundle/Redis/Producer.php36
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/rabbit.yml30
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/redis.yml81
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/services.yml33
-rw-r--r--tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php (renamed from tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php)10
-rw-r--r--tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php224
-rw-r--r--tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php19
-rw-r--r--tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php20
-rw-r--r--tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php20
-rw-r--r--tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php20
-rw-r--r--tests/Wallabag/ImportBundle/Import/PocketImportTest.php86
-rw-r--r--tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php45
-rw-r--r--tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php45
-rw-r--r--tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php41
28 files changed, 846 insertions, 68 deletions
diff --git a/app/config/config.yml b/app/config/config.yml
index d39bef95..e18a932d 100644
--- a/app/config/config.yml
+++ b/app/config/config.yml
@@ -254,7 +254,7 @@ old_sound_rabbit_mq:
254 type: topic 254 type: topic
255 queue_options: 255 queue_options:
256 name: 'wallabag.import.pocket' 256 name: 'wallabag.import.pocket'
257 callback: wallabag_import.consumer.pocket 257 callback: wallabag_import.consumer.ampq.pocket
258 import_readability: 258 import_readability:
259 connection: default 259 connection: default
260 exchange_options: 260 exchange_options:
@@ -262,7 +262,7 @@ old_sound_rabbit_mq:
262 type: topic 262 type: topic
263 queue_options: 263 queue_options:
264 name: 'wallabag.import.readability' 264 name: 'wallabag.import.readability'
265 callback: wallabag_import.consumer.readability 265 callback: wallabag_import.consumer.ampq.readability
266 import_wallabag_v1: 266 import_wallabag_v1:
267 connection: default 267 connection: default
268 exchange_options: 268 exchange_options:
@@ -270,7 +270,7 @@ old_sound_rabbit_mq:
270 type: topic 270 type: topic
271 queue_options: 271 queue_options:
272 name: 'wallabag.import.wallabag_v1' 272 name: 'wallabag.import.wallabag_v1'
273 callback: wallabag_import.consumer.wallabag_v1 273 callback: wallabag_import.consumer.ampq.wallabag_v1
274 import_wallabag_v2: 274 import_wallabag_v2:
275 connection: default 275 connection: default
276 exchange_options: 276 exchange_options:
@@ -278,4 +278,4 @@ old_sound_rabbit_mq:
278 type: topic 278 type: topic
279 queue_options: 279 queue_options:
280 name: 'wallabag.import.wallabag_v2' 280 name: 'wallabag.import.wallabag_v2'
281 callback: wallabag_import.consumer.wallabag_v2 281 callback: wallabag_import.consumer.ampq.wallabag_v2
diff --git a/app/config/parameters.yml.dist b/app/config/parameters.yml.dist
index a59dc02c..f2e5bec3 100644
--- a/app/config/parameters.yml.dist
+++ b/app/config/parameters.yml.dist
@@ -46,3 +46,7 @@ parameters:
46 rabbitmq_port: 5672 46 rabbitmq_port: 5672
47 rabbitmq_user: guest 47 rabbitmq_user: guest
48 rabbitmq_password: guest 48 rabbitmq_password: guest
49
50 # Redis processing
51 redis_host: localhost
52 redis_port: 6379
diff --git a/composer.json b/composer.json
index 121a3fd3..8220ed45 100644
--- a/composer.json
+++ b/composer.json
@@ -82,7 +82,9 @@
82 "ocramius/proxy-manager": "1.*", 82 "ocramius/proxy-manager": "1.*",
83 "white-october/pagerfanta-bundle": "^1.0", 83 "white-october/pagerfanta-bundle": "^1.0",
84 "mouf/nodejs-installer": "~1.0", 84 "mouf/nodejs-installer": "~1.0",
85 "php-amqplib/rabbitmq-bundle": "^1.8" 85 "php-amqplib/rabbitmq-bundle": "^1.8",
86 "predis/predis": "^1.0",
87 "javibravo/simpleue": "^1.0"
86 }, 88 },
87 "require-dev": { 89 "require-dev": {
88 "doctrine/doctrine-fixtures-bundle": "~2.2", 90 "doctrine/doctrine-fixtures-bundle": "~2.2",
@@ -90,7 +92,8 @@
90 "sensio/generator-bundle": "^3.0", 92 "sensio/generator-bundle": "^3.0",
91 "phpunit/phpunit": "~5.0", 93 "phpunit/phpunit": "~5.0",
92 "symfony/phpunit-bridge": "^3.0", 94 "symfony/phpunit-bridge": "^3.0",
93 "friendsofphp/php-cs-fixer": "~1.9" 95 "friendsofphp/php-cs-fixer": "~1.9",
96 "m6web/redis-mock": "^2.0"
94 }, 97 },
95 "scripts": { 98 "scripts": {
96 "post-cmd": [ 99 "post-cmd": [
diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php
index 8a2439ec..4b1ec02a 100644
--- a/src/Wallabag/CoreBundle/Command/InstallCommand.php
+++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php
@@ -322,7 +322,12 @@ class InstallCommand extends ContainerAwareCommand
322 'section' => 'import', 322 'section' => 'import',
323 ], 323 ],
324 [ 324 [
325 'name' => 'rabbitmq', 325 'name' => 'import_with_redis',
326 'value' => '0',
327 'section' => 'import',
328 ],
329 [
330 'name' => 'import_with_rabbitmq',
326 'value' => '0', 331 'value' => '0',
327 'section' => 'import', 332 'section' => 'import',
328 ], 333 ],
diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php
index 778f91ed..3ed6bafe 100644
--- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php
+++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php
@@ -96,7 +96,12 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface
96 'section' => 'import', 96 'section' => 'import',
97 ], 97 ],
98 [ 98 [
99 'name' => 'rabbitmq', 99 'name' => 'import_with_redis',
100 'value' => '0',
101 'section' => 'import',
102 ],
103 [
104 'name' => 'import_with_rabbitmq',
100 'value' => '0', 105 'value' => '0',
101 'section' => 'import', 106 'section' => 'import',
102 ], 107 ],
diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml
index e95ef452..23e6d3ca 100644
--- a/src/Wallabag/CoreBundle/Resources/config/services.yml
+++ b/src/Wallabag/CoreBundle/Resources/config/services.yml
@@ -125,3 +125,11 @@ services:
125 arguments: 125 arguments:
126 - "@security.token_storage" 126 - "@security.token_storage"
127 - "@router" 127 - "@router"
128
129 wallabag_core.redis.client:
130 class: Predis\Client
131 arguments:
132 -
133 host: '%redis_host%'
134 port: '%redis_port%'
135 schema: tcp
diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php
new file mode 100644
index 00000000..85c5a903
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php
@@ -0,0 +1,41 @@
1<?php
2
3namespace Wallabag\ImportBundle\Command;
4
5use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
6use Symfony\Component\Config\Definition\Exception\Exception;
7use Symfony\Component\Console\Input\InputArgument;
8use Symfony\Component\Console\Input\InputInterface;
9use Symfony\Component\Console\Output\OutputInterface;
10use Simpleue\Worker\QueueWorker;
11
12class RedisWorkerCommand extends ContainerAwareCommand
13{
14 protected function configure()
15 {
16 $this
17 ->setName('wallabag:import:redis-worker')
18 ->setDescription('Launch Redis worker')
19 ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
20 ;
21 }
22
23 protected function execute(InputInterface $input, OutputInterface $output)
24 {
25 $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
26 $output->writeln('Waiting for message ...');
27
28 $serviceName = $input->getArgument('serviceName');
29
30 if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
31 throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
32 }
33
34 $worker = new QueueWorker(
35 $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
36 $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName)
37 );
38
39 $worker->start();
40 }
41}
diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php
index 72a3260a..39bb5375 100644
--- a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php
+++ b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php
@@ -1,6 +1,6 @@
1<?php 1<?php
2 2
3namespace Wallabag\ImportBundle\Consumer\AMPQ; 3namespace Wallabag\ImportBundle\Consumer;
4 4
5use Doctrine\ORM\EntityManager; 5use Doctrine\ORM\EntityManager;
6use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface; 6use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
@@ -12,7 +12,7 @@ use Wallabag\CoreBundle\Entity\Tag;
12use Psr\Log\LoggerInterface; 12use Psr\Log\LoggerInterface;
13use Psr\Log\NullLogger; 13use Psr\Log\NullLogger;
14 14
15class EntryConsumer implements ConsumerInterface 15class AMPQEntryConsumer implements ConsumerInterface
16{ 16{
17 private $em; 17 private $em;
18 private $userRepository; 18 private $userRepository;
@@ -64,5 +64,7 @@ class EntryConsumer implements ConsumerInterface
64 64
65 return; 65 return;
66 } 66 }
67
68 $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
67 } 69 }
68} 70}
diff --git a/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php
new file mode 100644
index 00000000..38665b01
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php
@@ -0,0 +1,84 @@
1<?php
2
3namespace Wallabag\ImportBundle\Consumer;
4
5use Simpleue\Job\Job;
6use Doctrine\ORM\EntityManager;
7use Wallabag\ImportBundle\Import\AbstractImport;
8use Wallabag\UserBundle\Repository\UserRepository;
9use Wallabag\CoreBundle\Entity\Entry;
10use Wallabag\CoreBundle\Entity\Tag;
11use Psr\Log\LoggerInterface;
12use Psr\Log\NullLogger;
13
14class RedisEntryConsumer implements Job
15{
16 private $em;
17 private $userRepository;
18 private $import;
19 private $logger;
20
21 public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
22 {
23 $this->em = $em;
24 $this->userRepository = $userRepository;
25 $this->import = $import;
26 $this->logger = $logger ?: new NullLogger();
27 }
28
29 /**
30 * Handle one message by one message.
31 *
32 * @param string $job Content of the message (directly from Redis)
33 *
34 * @return bool
35 */
36 public function manage($job)
37 {
38 $storedEntry = json_decode($job, true);
39
40 $user = $this->userRepository->find($storedEntry['userId']);
41
42 // no user? Drop message
43 if (null === $user) {
44 $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
45
46 return false;
47 }
48
49 $this->import->setUser($user);
50
51 $entry = $this->import->parseEntry($storedEntry);
52
53 if (null === $entry) {
54 $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
55
56 return false;
57 }
58
59 try {
60 $this->em->flush();
61
62 // clear only affected entities
63 $this->em->clear(Entry::class);
64 $this->em->clear(Tag::class);
65 } catch (\Exception $e) {
66 $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
67
68 return false;
69 }
70
71 $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
72
73 return true;
74 }
75
76 /**
77 * Should tell if the given job will kill the worker.
78 * We don't want to stop it :).
79 */
80 public function isStopJob($job)
81 {
82 return false;
83 }
84}
diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php
index 57c007c3..3d555717 100644
--- a/src/Wallabag/ImportBundle/Controller/PocketController.php
+++ b/src/Wallabag/ImportBundle/Controller/PocketController.php
@@ -20,8 +20,10 @@ class PocketController extends Controller
20 $pocket = $this->get('wallabag_import.pocket.import'); 20 $pocket = $this->get('wallabag_import.pocket.import');
21 $pocket->setUser($this->getUser()); 21 $pocket->setUser($this->getUser());
22 22
23 if ($this->get('craue_config')->get('rabbitmq')) { 23 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
24 $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); 24 $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
25 } elseif ($this->get('craue_config')->get('import_with_redis')) {
26 $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
25 } 27 }
26 28
27 return $pocket; 29 return $pocket;
diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
index ee875a40..61243042 100644
--- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
+++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
@@ -20,8 +20,10 @@ class ReadabilityController extends Controller
20 $readability = $this->get('wallabag_import.readability.import'); 20 $readability = $this->get('wallabag_import.readability.import');
21 $readability->setUser($this->getUser()); 21 $readability->setUser($this->getUser());
22 22
23 if ($this->get('craue_config')->get('rabbitmq')) { 23 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
24 $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); 24 $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
25 } elseif ($this->get('craue_config')->get('import_with_redis')) {
26 $readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
25 } 27 }
26 28
27 if ($form->isValid()) { 29 if ($form->isValid()) {
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
index f80aec3a..312c7a35 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
@@ -14,8 +14,10 @@ class WallabagV1Controller extends WallabagController
14 { 14 {
15 $service = $this->get('wallabag_import.wallabag_v1.import'); 15 $service = $this->get('wallabag_import.wallabag_v1.import');
16 16
17 if ($this->get('craue_config')->get('rabbitmq')) { 17 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
18 $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); 18 $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
19 } elseif ($this->get('craue_config')->get('import_with_redis')) {
20 $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
19 } 21 }
20 22
21 return $service; 23 return $service;
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
index 063cddd9..45211fe6 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
@@ -14,8 +14,10 @@ class WallabagV2Controller extends WallabagController
14 { 14 {
15 $service = $this->get('wallabag_import.wallabag_v2.import'); 15 $service = $this->get('wallabag_import.wallabag_v2.import');
16 16
17 if ($this->get('craue_config')->get('rabbitmq')) { 17 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
18 $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); 18 $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
19 } elseif ($this->get('craue_config')->get('import_with_redis')) {
20 $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
19 } 21 }
20 22
21 return $service; 23 return $service;
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index 39befa7b..4cd8e846 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -9,7 +9,7 @@ use Wallabag\CoreBundle\Helper\ContentProxy;
9use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
10use Wallabag\CoreBundle\Entity\Tag; 10use Wallabag\CoreBundle\Entity\Tag;
11use Wallabag\UserBundle\Entity\User; 11use Wallabag\UserBundle\Entity\User;
12use OldSound\RabbitMqBundle\RabbitMq\Producer; 12use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
13 13
14abstract class AbstractImport implements ImportInterface 14abstract class AbstractImport implements ImportInterface
15{ 15{
@@ -35,12 +35,12 @@ abstract class AbstractImport implements ImportInterface
35 } 35 }
36 36
37 /** 37 /**
38 * Set RabbitMQ Producer to send each entry to a queue. 38 * Set RabbitMQ/Redis Producer to send each entry to a queue.
39 * This method should be called when user has enabled RabbitMQ. 39 * This method should be called when user has enabled RabbitMQ.
40 * 40 *
41 * @param Producer $producer 41 * @param ProducerInterface $producer
42 */ 42 */
43 public function setRabbitmqProducer(Producer $producer) 43 public function setProducer(ProducerInterface $producer)
44 { 44 {
45 $this->producer = $producer; 45 $this->producer = $producer;
46 } 46 }
diff --git a/src/Wallabag/ImportBundle/Redis/Producer.php b/src/Wallabag/ImportBundle/Redis/Producer.php
new file mode 100644
index 00000000..fedc3e57
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Redis/Producer.php
@@ -0,0 +1,36 @@
1<?php
2
3namespace Wallabag\ImportBundle\Redis;
4
5use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
6use Simpleue\Queue\RedisQueue;
7
8/**
9 * This is a proxy class for "Simpleue\Queue\RedisQueue".
10 * It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
11 *
12 * It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
13 * kind of class which implements the same interface.
14 * So we can inject either a RabbitMQ producer or a Redis producer with the same signature
15 */
16class Producer implements ProducerInterface
17{
18 private $queue;
19
20 public function __construct(RedisQueue $queue)
21 {
22 $this->queue = $queue;
23 }
24
25 /**
26 * Publish a message in the Redis queue.
27 *
28 * @param string $msgBody
29 * @param string $routingKey NOT USED
30 * @param array $additionalProperties NOT USED
31 */
32 public function publish($msgBody, $routingKey = '', $additionalProperties = array())
33 {
34 $this->queue->sendJob($msgBody);
35 }
36}
diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml
new file mode 100644
index 00000000..f09dda0d
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml
@@ -0,0 +1,30 @@
1# RabbitMQ stuff
2services:
3 wallabag_import.consumer.ampq.pocket:
4 class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
5 arguments:
6 - "@doctrine.orm.entity_manager"
7 - "@wallabag_user.user_repository"
8 - "@wallabag_import.pocket.import"
9 - "@logger"
10 wallabag_import.consumer.ampq.readability:
11 class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
12 arguments:
13 - "@doctrine.orm.entity_manager"
14 - "@wallabag_user.user_repository"
15 - "@wallabag_import.readability.import"
16 - "@logger"
17 wallabag_import.consumer.ampq.wallabag_v1:
18 class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
19 arguments:
20 - "@doctrine.orm.entity_manager"
21 - "@wallabag_user.user_repository"
22 - "@wallabag_import.wallabag_v1.import"
23 - "@logger"
24 wallabag_import.consumer.ampq.wallabag_v2:
25 class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
26 arguments:
27 - "@doctrine.orm.entity_manager"
28 - "@wallabag_user.user_repository"
29 - "@wallabag_import.wallabag_v2.import"
30 - "@logger"
diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml
new file mode 100644
index 00000000..7d3248e5
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml
@@ -0,0 +1,81 @@
1# Redis stuff
2services:
3 # readability
4 wallabag_import.queue.redis.readability:
5 class: Simpleue\Queue\RedisQueue
6 arguments:
7 - "@wallabag_core.redis.client"
8 - "wallabag.import.readability"
9
10 wallabag_import.producer.redis.readability:
11 class: Wallabag\ImportBundle\Redis\Producer
12 arguments:
13 - "@wallabag_import.queue.redis.readability"
14
15 wallabag_import.consumer.redis.readability:
16 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
17 arguments:
18 - "@doctrine.orm.entity_manager"
19 - "@wallabag_user.user_repository"
20 - "@wallabag_import.readability.import"
21 - "@logger"
22
23 # pocket
24 wallabag_import.queue.redis.pocket:
25 class: Simpleue\Queue\RedisQueue
26 arguments:
27 - "@wallabag_core.redis.client"
28 - "wallabag.import.pocket"
29
30 wallabag_import.producer.redis.pocket:
31 class: Wallabag\ImportBundle\Redis\Producer
32 arguments:
33 - "@wallabag_import.queue.redis.pocket"
34
35 wallabag_import.consumer.redis.pocket:
36 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
37 arguments:
38 - "@doctrine.orm.entity_manager"
39 - "@wallabag_user.user_repository"
40 - "@wallabag_import.pocket.import"
41 - "@logger"
42
43 # wallabag v1
44 wallabag_import.queue.redis.wallabag_v1:
45 class: Simpleue\Queue\RedisQueue
46 arguments:
47 - "@wallabag_core.redis.client"
48 - "wallabag.import.wallabag_v1"
49
50 wallabag_import.producer.redis.wallabag_v1:
51 class: Wallabag\ImportBundle\Redis\Producer
52 arguments:
53 - "@wallabag_import.queue.redis.wallabag_v1"
54
55 wallabag_import.consumer.redis.wallabag_v1:
56 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
57 arguments:
58 - "@doctrine.orm.entity_manager"
59 - "@wallabag_user.user_repository"
60 - "@wallabag_import.wallabag_v1.import"
61 - "@logger"
62
63 # wallabag v2
64 wallabag_import.queue.redis.wallabag_v2:
65 class: Simpleue\Queue\RedisQueue
66 arguments:
67 - "@wallabag_core.redis.client"
68 - "wallabag.import.wallabag_v2"
69
70 wallabag_import.producer.redis.wallabag_v2:
71 class: Wallabag\ImportBundle\Redis\Producer
72 arguments:
73 - "@wallabag_import.queue.redis.wallabag_v2"
74
75 wallabag_import.consumer.redis.wallabag_v2:
76 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
77 arguments:
78 - "@doctrine.orm.entity_manager"
79 - "@wallabag_user.user_repository"
80 - "@wallabag_import.wallabag_v2.import"
81 - "@logger"
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index cad44e71..f03404ae 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -1,33 +1,8 @@
1services: 1imports:
2 wallabag_import.consumer.pocket: 2 - { resource: rabbit.yml }
3 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer 3 - { resource: redis.yml }
4 arguments:
5 - "@doctrine.orm.entity_manager"
6 - "@wallabag_user.user_repository"
7 - "@wallabag_import.pocket.import"
8 - "@logger"
9 wallabag_import.consumer.readability:
10 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
11 arguments:
12 - "@doctrine.orm.entity_manager"
13 - "@wallabag_user.user_repository"
14 - "@wallabag_import.readability.import"
15 - "@logger"
16 wallabag_import.consumer.wallabag_v1:
17 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
18 arguments:
19 - "@doctrine.orm.entity_manager"
20 - "@wallabag_user.user_repository"
21 - "@wallabag_import.wallabag_v1.import"
22 - "@logger"
23 wallabag_import.consumer.wallabag_v2:
24 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
25 arguments:
26 - "@doctrine.orm.entity_manager"
27 - "@wallabag_user.user_repository"
28 - "@wallabag_import.wallabag_v2.import"
29 - "@logger"
30 4
5services:
31 wallabag_import.chain: 6 wallabag_import.chain:
32 class: Wallabag\ImportBundle\Import\ImportChain 7 class: Wallabag\ImportBundle\Import\ImportChain
33 8
diff --git a/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php
index 7141874c..b13ade1d 100644
--- a/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php
+++ b/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php
@@ -2,12 +2,12 @@
2 2
3namespace Tests\Wallabag\ImportBundle\Consumer\AMQP; 3namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
4 4
5use Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer; 5use Wallabag\ImportBundle\Consumer\AMPQEntryConsumer;
6use PhpAmqpLib\Message\AMQPMessage; 6use PhpAmqpLib\Message\AMQPMessage;
7use Wallabag\UserBundle\Entity\User; 7use Wallabag\UserBundle\Entity\User;
8use Wallabag\CoreBundle\Entity\Entry; 8use Wallabag\CoreBundle\Entity\Entry;
9 9
10class EntryConsumerTest extends \PHPUnit_Framework_TestCase 10class AMPQEntryConsumerTest extends \PHPUnit_Framework_TestCase
11{ 11{
12 public function testMessageOk() 12 public function testMessageOk()
13 { 13 {
@@ -112,7 +112,7 @@ JSON;
112 ->with(json_decode($body, true)) 112 ->with(json_decode($body, true))
113 ->willReturn($entry); 113 ->willReturn($entry);
114 114
115 $consumer = new EntryConsumer( 115 $consumer = new AMPQEntryConsumer(
116 $em, 116 $em,
117 $userRepository, 117 $userRepository,
118 $import 118 $import
@@ -157,7 +157,7 @@ JSON;
157 ->disableOriginalConstructor() 157 ->disableOriginalConstructor()
158 ->getMock(); 158 ->getMock();
159 159
160 $consumer = new EntryConsumer( 160 $consumer = new AMPQEntryConsumer(
161 $em, 161 $em,
162 $userRepository, 162 $userRepository,
163 $import 163 $import
@@ -212,7 +212,7 @@ JSON;
212 ->with(json_decode($body, true)) 212 ->with(json_decode($body, true))
213 ->willReturn(null); 213 ->willReturn(null);
214 214
215 $consumer = new EntryConsumer( 215 $consumer = new AMPQEntryConsumer(
216 $em, 216 $em,
217 $userRepository, 217 $userRepository,
218 $import 218 $import
diff --git a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php
new file mode 100644
index 00000000..0ce7ce49
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php
@@ -0,0 +1,224 @@
1<?php
2
3namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
4
5use Wallabag\ImportBundle\Consumer\RedisEntryConsumer;
6use Wallabag\UserBundle\Entity\User;
7use Wallabag\CoreBundle\Entity\Entry;
8
9class RedisEntryConsumerTest extends \PHPUnit_Framework_TestCase
10{
11 public function testMessageOk()
12 {
13 $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
14 ->disableOriginalConstructor()
15 ->getMock();
16
17 $em
18 ->expects($this->once())
19 ->method('flush');
20
21 $em
22 ->expects($this->exactly(2))
23 ->method('clear');
24
25 $body = <<<'JSON'
26{
27 "item_id": "1402935436",
28 "resolved_id": "1402935436",
29 "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
30 "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
31 "favorite": "0",
32 "status": "0",
33 "time_added": "1473020899",
34 "time_updated": "1473020899",
35 "time_read": "0",
36 "time_favorited": "0",
37 "sort_id": 0,
38 "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
39 "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
40 "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...",
41 "is_article": "1",
42 "is_index": "0",
43 "has_video": "0",
44 "has_image": "1",
45 "word_count": "200",
46 "tags": {
47 "ifttt": {
48 "item_id": "1402935436",
49 "tag": "ifttt"
50 },
51 "mashable": {
52 "item_id": "1402935436",
53 "tag": "mashable"
54 }
55 },
56 "authors": {
57 "2484273": {
58 "item_id": "1402935436",
59 "author_id": "2484273",
60 "name": "Adam Rosenberg",
61 "url": "http://mashable.com/author/adam-rosenberg/"
62 }
63 },
64 "image": {
65 "item_id": "1402935436",
66 "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
67 "width": "0",
68 "height": "0"
69 },
70 "images": {
71 "1": {
72 "item_id": "1402935436",
73 "image_id": "1",
74 "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
75 "width": "0",
76 "height": "0",
77 "credit": "Image: Steve Eichner/NameFace/Sipa USA",
78 "caption": ""
79 }
80 },
81 "userId": 1
82}
83JSON;
84
85 $user = new User();
86 $entry = new Entry($user);
87
88 $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
89 ->disableOriginalConstructor()
90 ->getMock();
91
92 $userRepository
93 ->expects($this->once())
94 ->method('find')
95 // userId from the body json above
96 ->with(1)
97 ->willReturn($user);
98
99 $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
100 ->disableOriginalConstructor()
101 ->getMock();
102
103 $import
104 ->expects($this->once())
105 ->method('setUser')
106 ->with($user);
107
108 $import
109 ->expects($this->once())
110 ->method('parseEntry')
111 ->with(json_decode($body, true))
112 ->willReturn($entry);
113
114 $consumer = new RedisEntryConsumer(
115 $em,
116 $userRepository,
117 $import
118 );
119
120 $res = $consumer->manage($body);
121
122 $this->assertTrue($res);
123 }
124
125 public function testMessageWithBadUser()
126 {
127 $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
128 ->disableOriginalConstructor()
129 ->getMock();
130
131 $em
132 ->expects($this->never())
133 ->method('flush');
134
135 $em
136 ->expects($this->never())
137 ->method('clear');
138
139 $body = '{ "userId": 123 }';
140
141 $user = new User();
142 $entry = new Entry($user);
143
144 $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
145 ->disableOriginalConstructor()
146 ->getMock();
147
148 $userRepository
149 ->expects($this->once())
150 ->method('find')
151 // userId from the body json above
152 ->with(123)
153 ->willReturn(null);
154
155 $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
156 ->disableOriginalConstructor()
157 ->getMock();
158
159 $consumer = new RedisEntryConsumer(
160 $em,
161 $userRepository,
162 $import
163 );
164
165 $res = $consumer->manage($body);
166
167 $this->assertFalse($res);
168 }
169
170 public function testMessageWithEntryProcessed()
171 {
172 $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
173 ->disableOriginalConstructor()
174 ->getMock();
175
176 $em
177 ->expects($this->never())
178 ->method('flush');
179
180 $em
181 ->expects($this->never())
182 ->method('clear');
183
184 $body = '{ "userId": 123 }';
185
186 $user = new User();
187
188 $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
189 ->disableOriginalConstructor()
190 ->getMock();
191
192 $userRepository
193 ->expects($this->once())
194 ->method('find')
195 // userId from the body json above
196 ->with(123)
197 ->willReturn($user);
198
199 $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
200 ->disableOriginalConstructor()
201 ->getMock();
202
203 $import
204 ->expects($this->once())
205 ->method('setUser')
206 ->with($user);
207
208 $import
209 ->expects($this->once())
210 ->method('parseEntry')
211 ->with(json_decode($body, true))
212 ->willReturn(null);
213
214 $consumer = new RedisEntryConsumer(
215 $em,
216 $userRepository,
217 $import
218 );
219
220 $res = $consumer->manage($body);
221
222 $this->assertFalse($res);
223 }
224}
diff --git a/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php
index 098cf356..35673261 100644
--- a/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php
+++ b/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php
@@ -22,14 +22,29 @@ class PocketControllerTest extends WallabagCoreTestCase
22 $this->logInAs('admin'); 22 $this->logInAs('admin');
23 $client = $this->getClient(); 23 $client = $this->getClient();
24 24
25 $client->getContainer()->get('craue_config')->set('rabbitmq', 1); 25 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
26 26
27 $crawler = $client->request('GET', '/import/pocket'); 27 $crawler = $client->request('GET', '/import/pocket');
28 28
29 $this->assertEquals(200, $client->getResponse()->getStatusCode()); 29 $this->assertEquals(200, $client->getResponse()->getStatusCode());
30 $this->assertEquals(1, $crawler->filter('button[type=submit]')->count()); 30 $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
31 31
32 $client->getContainer()->get('craue_config')->set('rabbitmq', 0); 32 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
33 }
34
35 public function testImportPocketWithRedisEnabled()
36 {
37 $this->logInAs('admin');
38 $client = $this->getClient();
39
40 $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
41
42 $crawler = $client->request('GET', '/import/pocket');
43
44 $this->assertEquals(200, $client->getResponse()->getStatusCode());
45 $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
46
47 $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
33 } 48 }
34 49
35 public function testImportPocketAuthBadToken() 50 public function testImportPocketAuthBadToken()
diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php
index e12a723d..69635382 100644
--- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php
+++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php
@@ -24,7 +24,7 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
24 $this->logInAs('admin'); 24 $this->logInAs('admin');
25 $client = $this->getClient(); 25 $client = $this->getClient();
26 26
27 $client->getContainer()->get('craue_config')->set('rabbitmq', 1); 27 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
28 28
29 $crawler = $client->request('GET', '/import/readability'); 29 $crawler = $client->request('GET', '/import/readability');
30 30
@@ -32,7 +32,23 @@ class ReadabilityControllerTest extends WallabagCoreTestCase
32 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); 32 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
33 $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); 33 $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
34 34
35 $client->getContainer()->get('craue_config')->set('rabbitmq', 0); 35 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
36 }
37
38 public function testImportReadabilityWithRedisEnabled()
39 {
40 $this->logInAs('admin');
41 $client = $this->getClient();
42
43 $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
44
45 $crawler = $client->request('GET', '/import/readability');
46
47 $this->assertEquals(200, $client->getResponse()->getStatusCode());
48 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
49 $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
50
51 $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
36 } 52 }
37 53
38 public function testImportReadabilityWithFile() 54 public function testImportReadabilityWithFile()
diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php
index 96556717..933ddd6c 100644
--- a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php
+++ b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php
@@ -24,7 +24,7 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
24 $this->logInAs('admin'); 24 $this->logInAs('admin');
25 $client = $this->getClient(); 25 $client = $this->getClient();
26 26
27 $client->getContainer()->get('craue_config')->set('rabbitmq', 1); 27 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
28 28
29 $crawler = $client->request('GET', '/import/wallabag-v1'); 29 $crawler = $client->request('GET', '/import/wallabag-v1');
30 30
@@ -32,7 +32,23 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase
32 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); 32 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
33 $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); 33 $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
34 34
35 $client->getContainer()->get('craue_config')->set('rabbitmq', 0); 35 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
36 }
37
38 public function testImportWallabagWithRedisEnabled()
39 {
40 $this->logInAs('admin');
41 $client = $this->getClient();
42
43 $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
44
45 $crawler = $client->request('GET', '/import/wallabag-v1');
46
47 $this->assertEquals(200, $client->getResponse()->getStatusCode());
48 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
49 $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
50
51 $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
36 } 52 }
37 53
38 public function testImportWallabagWithFile() 54 public function testImportWallabagWithFile()
diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php
index 250d0d3e..36e5221d 100644
--- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php
+++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php
@@ -24,7 +24,7 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
24 $this->logInAs('admin'); 24 $this->logInAs('admin');
25 $client = $this->getClient(); 25 $client = $this->getClient();
26 26
27 $client->getContainer()->get('craue_config')->set('rabbitmq', 1); 27 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
28 28
29 $crawler = $client->request('GET', '/import/wallabag-v2'); 29 $crawler = $client->request('GET', '/import/wallabag-v2');
30 30
@@ -32,7 +32,23 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase
32 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); 32 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
33 $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); 33 $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
34 34
35 $client->getContainer()->get('craue_config')->set('rabbitmq', 0); 35 $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
36 }
37
38 public function testImportWallabagWithRedisEnabled()
39 {
40 $this->logInAs('admin');
41 $client = $this->getClient();
42
43 $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
44
45 $crawler = $client->request('GET', '/import/wallabag-v2');
46
47 $this->assertEquals(200, $client->getResponse()->getStatusCode());
48 $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
49 $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
50
51 $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
36 } 52 }
37 53
38 public function testImportWallabagWithFile() 54 public function testImportWallabagWithFile()
diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php
index 1750e3a1..425fa321 100644
--- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php
+++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php
@@ -9,8 +9,11 @@ use GuzzleHttp\Client;
9use GuzzleHttp\Subscriber\Mock; 9use GuzzleHttp\Subscriber\Mock;
10use GuzzleHttp\Message\Response; 10use GuzzleHttp\Message\Response;
11use GuzzleHttp\Stream\Stream; 11use GuzzleHttp\Stream\Stream;
12use Wallabag\ImportBundle\Redis\Producer;
12use Monolog\Logger; 13use Monolog\Logger;
13use Monolog\Handler\TestHandler; 14use Monolog\Handler\TestHandler;
15use Simpleue\Queue\RedisQueue;
16use M6Web\Component\RedisMock\RedisMockFactory;
14 17
15class PocketImportTest extends \PHPUnit_Framework_TestCase 18class PocketImportTest extends \PHPUnit_Framework_TestCase
16{ 19{
@@ -442,7 +445,7 @@ JSON;
442 ->with(json_encode($bodyAsArray)); 445 ->with(json_encode($bodyAsArray));
443 446
444 $pocketImport->setClient($client); 447 $pocketImport->setClient($client);
445 $pocketImport->setRabbitmqProducer($producer); 448 $pocketImport->setProducer($producer);
446 $pocketImport->authorize('wunderbar_code'); 449 $pocketImport->authorize('wunderbar_code');
447 450
448 $res = $pocketImport->setMarkAsRead(true)->import(); 451 $res = $pocketImport->setMarkAsRead(true)->import();
@@ -451,6 +454,87 @@ JSON;
451 $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary()); 454 $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
452 } 455 }
453 456
457 /**
458 * Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
459 */
460 public function testImportWithRedis()
461 {
462 $client = new Client();
463
464 $body = <<<'JSON'
465{
466 "item_id": "229279689",
467 "resolved_id": "229279689",
468 "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
469 "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
470 "favorite": "1",
471 "status": "1",
472 "time_added": "1473020899",
473 "time_updated": "1473020899",
474 "time_read": "0",
475 "time_favorited": "0",
476 "sort_id": 0,
477 "resolved_title": "The Massive Ryder Cup Preview",
478 "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
479 "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
480 "is_article": "1",
481 "has_video": "0",
482 "has_image": "0",
483 "word_count": "3197"
484}
485JSON;
486
487 $mock = new Mock([
488 new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
489 new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
490 {
491 "status": 1,
492 "list": {
493 "229279690": '.$body.'
494 }
495 }
496 ')),
497 ]);
498
499 $client->getEmitter()->attach($mock);
500
501 $pocketImport = $this->getPocketImport();
502
503 $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
504 ->disableOriginalConstructor()
505 ->getMock();
506
507 $entryRepo->expects($this->never())
508 ->method('findByUrlAndUserId');
509
510 $this->em
511 ->expects($this->never())
512 ->method('getRepository');
513
514 $entry = new Entry($this->user);
515
516 $this->contentProxy
517 ->expects($this->never())
518 ->method('updateEntry');
519
520 $factory = new RedisMockFactory();
521 $redisMock = $factory->getAdapter('Predis\Client', true);
522
523 $queue = new RedisQueue($redisMock, 'pocket');
524 $producer = new Producer($queue);
525
526 $pocketImport->setClient($client);
527 $pocketImport->setProducer($producer);
528 $pocketImport->authorize('wunderbar_code');
529
530 $res = $pocketImport->setMarkAsRead(true)->import();
531
532 $this->assertTrue($res);
533 $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
534
535 $this->assertNotEmpty($redisMock->lpop('pocket'));
536 }
537
454 public function testImportBadResponse() 538 public function testImportBadResponse()
455 { 539 {
456 $client = new Client(); 540 $client = new Client();
diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php
index 69a66d6a..0981eedb 100644
--- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php
+++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php
@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
5use Wallabag\ImportBundle\Import\ReadabilityImport; 5use Wallabag\ImportBundle\Import\ReadabilityImport;
6use Wallabag\UserBundle\Entity\User; 6use Wallabag\UserBundle\Entity\User;
7use Wallabag\CoreBundle\Entity\Entry; 7use Wallabag\CoreBundle\Entity\Entry;
8use Wallabag\ImportBundle\Redis\Producer;
8use Monolog\Logger; 9use Monolog\Logger;
9use Monolog\Handler\TestHandler; 10use Monolog\Handler\TestHandler;
11use Simpleue\Queue\RedisQueue;
12use M6Web\Component\RedisMock\RedisMockFactory;
10 13
11class ReadabilityImportTest extends \PHPUnit_Framework_TestCase 14class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
12{ 15{
@@ -152,7 +155,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
152 ->expects($this->exactly(2)) 155 ->expects($this->exactly(2))
153 ->method('publish'); 156 ->method('publish');
154 157
155 $readabilityImport->setRabbitmqProducer($producer); 158 $readabilityImport->setProducer($producer);
156 159
157 $res = $readabilityImport->setMarkAsRead(true)->import(); 160 $res = $readabilityImport->setMarkAsRead(true)->import();
158 161
@@ -160,6 +163,46 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
160 $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); 163 $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
161 } 164 }
162 165
166 public function testImportWithRedis()
167 {
168 $readabilityImport = $this->getReadabilityImport();
169 $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
170
171 $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
172 ->disableOriginalConstructor()
173 ->getMock();
174
175 $entryRepo->expects($this->never())
176 ->method('findByUrlAndUserId');
177
178 $this->em
179 ->expects($this->never())
180 ->method('getRepository');
181
182 $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
183 ->disableOriginalConstructor()
184 ->getMock();
185
186 $this->contentProxy
187 ->expects($this->never())
188 ->method('updateEntry');
189
190 $factory = new RedisMockFactory();
191 $redisMock = $factory->getAdapter('Predis\Client', true);
192
193 $queue = new RedisQueue($redisMock, 'readability');
194 $producer = new Producer($queue);
195
196 $readabilityImport->setProducer($producer);
197
198 $res = $readabilityImport->setMarkAsRead(true)->import();
199
200 $this->assertTrue($res);
201 $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
202
203 $this->assertNotEmpty($redisMock->lpop('readability'));
204 }
205
163 public function testImportBadFile() 206 public function testImportBadFile()
164 { 207 {
165 $readabilityImport = $this->getReadabilityImport(); 208 $readabilityImport = $this->getReadabilityImport();
diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
index ada5493e..b43682cd 100644
--- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
+++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
5use Wallabag\ImportBundle\Import\WallabagV1Import; 5use Wallabag\ImportBundle\Import\WallabagV1Import;
6use Wallabag\UserBundle\Entity\User; 6use Wallabag\UserBundle\Entity\User;
7use Wallabag\CoreBundle\Entity\Entry; 7use Wallabag\CoreBundle\Entity\Entry;
8use Wallabag\ImportBundle\Redis\Producer;
8use Monolog\Logger; 9use Monolog\Logger;
9use Monolog\Handler\TestHandler; 10use Monolog\Handler\TestHandler;
11use Simpleue\Queue\RedisQueue;
12use M6Web\Component\RedisMock\RedisMockFactory;
10 13
11class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase 14class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
12{ 15{
@@ -152,7 +155,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
152 ->expects($this->exactly(4)) 155 ->expects($this->exactly(4))
153 ->method('publish'); 156 ->method('publish');
154 157
155 $wallabagV1Import->setRabbitmqProducer($producer); 158 $wallabagV1Import->setProducer($producer);
156 159
157 $res = $wallabagV1Import->setMarkAsRead(true)->import(); 160 $res = $wallabagV1Import->setMarkAsRead(true)->import();
158 161
@@ -160,6 +163,46 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
160 $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary()); 163 $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
161 } 164 }
162 165
166 public function testImportWithRedis()
167 {
168 $wallabagV1Import = $this->getWallabagV1Import();
169 $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
170
171 $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
172 ->disableOriginalConstructor()
173 ->getMock();
174
175 $entryRepo->expects($this->never())
176 ->method('findByUrlAndUserId');
177
178 $this->em
179 ->expects($this->never())
180 ->method('getRepository');
181
182 $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
183 ->disableOriginalConstructor()
184 ->getMock();
185
186 $this->contentProxy
187 ->expects($this->never())
188 ->method('updateEntry');
189
190 $factory = new RedisMockFactory();
191 $redisMock = $factory->getAdapter('Predis\Client', true);
192
193 $queue = new RedisQueue($redisMock, 'wallabag_v1');
194 $producer = new Producer($queue);
195
196 $wallabagV1Import->setProducer($producer);
197
198 $res = $wallabagV1Import->setMarkAsRead(true)->import();
199
200 $this->assertTrue($res);
201 $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
202
203 $this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
204 }
205
163 public function testImportBadFile() 206 public function testImportBadFile()
164 { 207 {
165 $wallabagV1Import = $this->getWallabagV1Import(); 208 $wallabagV1Import = $this->getWallabagV1Import();
diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php
index 51f0aada..18998b35 100644
--- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php
+++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php
@@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import;
5use Wallabag\ImportBundle\Import\WallabagV2Import; 5use Wallabag\ImportBundle\Import\WallabagV2Import;
6use Wallabag\UserBundle\Entity\User; 6use Wallabag\UserBundle\Entity\User;
7use Wallabag\CoreBundle\Entity\Entry; 7use Wallabag\CoreBundle\Entity\Entry;
8use Wallabag\ImportBundle\Redis\Producer;
8use Monolog\Logger; 9use Monolog\Logger;
9use Monolog\Handler\TestHandler; 10use Monolog\Handler\TestHandler;
11use Simpleue\Queue\RedisQueue;
12use M6Web\Component\RedisMock\RedisMockFactory;
10 13
11class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase 14class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
12{ 15{
@@ -144,7 +147,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
144 ->expects($this->exactly(24)) 147 ->expects($this->exactly(24))
145 ->method('publish'); 148 ->method('publish');
146 149
147 $wallabagV2Import->setRabbitmqProducer($producer); 150 $wallabagV2Import->setProducer($producer);
148 151
149 $res = $wallabagV2Import->setMarkAsRead(true)->import(); 152 $res = $wallabagV2Import->setMarkAsRead(true)->import();
150 153
@@ -152,6 +155,42 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
152 $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary()); 155 $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
153 } 156 }
154 157
158 public function testImportWithRedis()
159 {
160 $wallabagV2Import = $this->getWallabagV2Import();
161 $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
162
163 $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
164 ->disableOriginalConstructor()
165 ->getMock();
166
167 $entryRepo->expects($this->never())
168 ->method('findByUrlAndUserId');
169
170 $this->em
171 ->expects($this->never())
172 ->method('getRepository');
173
174 $this->contentProxy
175 ->expects($this->never())
176 ->method('updateEntry');
177
178 $factory = new RedisMockFactory();
179 $redisMock = $factory->getAdapter('Predis\Client', true);
180
181 $queue = new RedisQueue($redisMock, 'wallabag_v2');
182 $producer = new Producer($queue);
183
184 $wallabagV2Import->setProducer($producer);
185
186 $res = $wallabagV2Import->setMarkAsRead(true)->import();
187
188 $this->assertTrue($res);
189 $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
190
191 $this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
192 }
193
155 public function testImportBadFile() 194 public function testImportBadFile()
156 { 195 {
157 $wallabagV1Import = $this->getWallabagV2Import(); 196 $wallabagV1Import = $this->getWallabagV2Import();