type: topic
queue_options:
name: 'wallabag.import.pocket'
- callback: wallabag_import.consumer.pocket
+ callback: wallabag_import.consumer.ampq.pocket
import_readability:
connection: default
exchange_options:
type: topic
queue_options:
name: 'wallabag.import.readability'
- callback: wallabag_import.consumer.readability
+ callback: wallabag_import.consumer.ampq.readability
import_wallabag_v1:
connection: default
exchange_options:
type: topic
queue_options:
name: 'wallabag.import.wallabag_v1'
- callback: wallabag_import.consumer.wallabag_v1
+ callback: wallabag_import.consumer.ampq.wallabag_v1
import_wallabag_v2:
connection: default
exchange_options:
type: topic
queue_options:
name: 'wallabag.import.wallabag_v2'
- callback: wallabag_import.consumer.wallabag_v2
+ callback: wallabag_import.consumer.ampq.wallabag_v2
rabbitmq_port: 5672
rabbitmq_user: guest
rabbitmq_password: guest
+
+ # Redis processing
+ redis_host: localhost
+ redis_port: 6379
"ocramius/proxy-manager": "1.*",
"white-october/pagerfanta-bundle": "^1.0",
"mouf/nodejs-installer": "~1.0",
- "php-amqplib/rabbitmq-bundle": "^1.8"
+ "php-amqplib/rabbitmq-bundle": "^1.8",
+ "predis/predis": "^1.0",
+ "javibravo/simpleue": "^1.0"
},
"require-dev": {
"doctrine/doctrine-fixtures-bundle": "~2.2",
"sensio/generator-bundle": "^3.0",
"phpunit/phpunit": "~5.0",
"symfony/phpunit-bridge": "^3.0",
- "friendsofphp/php-cs-fixer": "~1.9"
+ "friendsofphp/php-cs-fixer": "~1.9",
+ "m6web/redis-mock": "^2.0"
},
"scripts": {
"post-cmd": [
'section' => 'import',
],
[
- 'name' => 'rabbitmq',
+ 'name' => 'import_with_redis',
+ 'value' => '0',
+ 'section' => 'import',
+ ],
+ [
+ 'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],
'section' => 'import',
],
[
- 'name' => 'rabbitmq',
+ 'name' => 'import_with_redis',
+ 'value' => '0',
+ 'section' => 'import',
+ ],
+ [
+ 'name' => 'import_with_rabbitmq',
'value' => '0',
'section' => 'import',
],
arguments:
- "@security.token_storage"
- "@router"
+
+ wallabag_core.redis.client:
+ class: Predis\Client
+ arguments:
+ -
+ host: '%redis_host%'
+ port: '%redis_port%'
+ schema: tcp
--- /dev/null
+<?php
+
+namespace Wallabag\ImportBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Config\Definition\Exception\Exception;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Simpleue\Worker\QueueWorker;
+
+class RedisWorkerCommand extends ContainerAwareCommand
+{
+ protected function configure()
+ {
+ $this
+ ->setName('wallabag:import:redis-worker')
+ ->setDescription('Launch Redis worker')
+ ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
+ ;
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
+ $output->writeln('Waiting for message ...');
+
+ $serviceName = $input->getArgument('serviceName');
+
+ if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
+ throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
+ }
+
+ $worker = new QueueWorker(
+ $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
+ $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName)
+ );
+
+ $worker->start();
+ }
+}
<?php
-namespace Wallabag\ImportBundle\Consumer\AMPQ;
+namespace Wallabag\ImportBundle\Consumer;
use Doctrine\ORM\EntityManager;
use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
-class EntryConsumer implements ConsumerInterface
+class AMPQEntryConsumer implements ConsumerInterface
{
private $em;
private $userRepository;
return;
}
+
+ $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
}
}
--- /dev/null
+<?php
+
+namespace Wallabag\ImportBundle\Consumer;
+
+use Simpleue\Job\Job;
+use Doctrine\ORM\EntityManager;
+use Wallabag\ImportBundle\Import\AbstractImport;
+use Wallabag\UserBundle\Repository\UserRepository;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\CoreBundle\Entity\Tag;
+use Psr\Log\LoggerInterface;
+use Psr\Log\NullLogger;
+
+class RedisEntryConsumer implements Job
+{
+ private $em;
+ private $userRepository;
+ private $import;
+ private $logger;
+
+ public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
+ {
+ $this->em = $em;
+ $this->userRepository = $userRepository;
+ $this->import = $import;
+ $this->logger = $logger ?: new NullLogger();
+ }
+
+ /**
+ * Handle one message by one message.
+ *
+ * @param string $job Content of the message (directly from Redis)
+ *
+ * @return bool
+ */
+ public function manage($job)
+ {
+ $storedEntry = json_decode($job, true);
+
+ $user = $this->userRepository->find($storedEntry['userId']);
+
+ // no user? Drop message
+ if (null === $user) {
+ $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
+
+ return false;
+ }
+
+ $this->import->setUser($user);
+
+ $entry = $this->import->parseEntry($storedEntry);
+
+ if (null === $entry) {
+ $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
+
+ return false;
+ }
+
+ try {
+ $this->em->flush();
+
+ // clear only affected entities
+ $this->em->clear(Entry::class);
+ $this->em->clear(Tag::class);
+ } catch (\Exception $e) {
+ $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
+
+ return false;
+ }
+
+ $this->logger->info('Content with url ('.$entry->getUrl().') imported !');
+
+ return true;
+ }
+
+ /**
+ * Should tell if the given job will kill the worker.
+ * We don't want to stop it :).
+ */
+ public function isStopJob($job)
+ {
+ return false;
+ }
+}
$pocket = $this->get('wallabag_import.pocket.import');
$pocket->setUser($this->getUser());
- if ($this->get('craue_config')->get('rabbitmq')) {
- $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+ if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+ $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
+ } elseif ($this->get('craue_config')->get('import_with_redis')) {
+ $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
}
return $pocket;
$readability = $this->get('wallabag_import.readability.import');
$readability->setUser($this->getUser());
- if ($this->get('craue_config')->get('rabbitmq')) {
- $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+ if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+ $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
+ } elseif ($this->get('craue_config')->get('import_with_redis')) {
+ $readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
}
if ($form->isValid()) {
{
$service = $this->get('wallabag_import.wallabag_v1.import');
- if ($this->get('craue_config')->get('rabbitmq')) {
- $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+ if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+ $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
+ } elseif ($this->get('craue_config')->get('import_with_redis')) {
+ $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
}
return $service;
{
$service = $this->get('wallabag_import.wallabag_v2.import');
- if ($this->get('craue_config')->get('rabbitmq')) {
- $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+ if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+ $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
+ } elseif ($this->get('craue_config')->get('import_with_redis')) {
+ $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
}
return $service;
use Wallabag\CoreBundle\Entity\Entry;
use Wallabag\CoreBundle\Entity\Tag;
use Wallabag\UserBundle\Entity\User;
-use OldSound\RabbitMqBundle\RabbitMq\Producer;
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
abstract class AbstractImport implements ImportInterface
{
}
/**
- * Set RabbitMQ Producer to send each entry to a queue.
+ * Set RabbitMQ/Redis Producer to send each entry to a queue.
* This method should be called when user has enabled RabbitMQ.
*
- * @param Producer $producer
+ * @param ProducerInterface $producer
*/
- public function setRabbitmqProducer(Producer $producer)
+ public function setProducer(ProducerInterface $producer)
{
$this->producer = $producer;
}
--- /dev/null
+<?php
+
+namespace Wallabag\ImportBundle\Redis;
+
+use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
+use Simpleue\Queue\RedisQueue;
+
+/**
+ * This is a proxy class for "Simpleue\Queue\RedisQueue".
+ * It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
+ *
+ * It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
+ * kind of class which implements the same interface.
+ * So we can inject either a RabbitMQ producer or a Redis producer with the same signature
+ */
+class Producer implements ProducerInterface
+{
+ private $queue;
+
+ public function __construct(RedisQueue $queue)
+ {
+ $this->queue = $queue;
+ }
+
+ /**
+ * Publish a message in the Redis queue.
+ *
+ * @param string $msgBody
+ * @param string $routingKey NOT USED
+ * @param array $additionalProperties NOT USED
+ */
+ public function publish($msgBody, $routingKey = '', $additionalProperties = array())
+ {
+ $this->queue->sendJob($msgBody);
+ }
+}
--- /dev/null
+# RabbitMQ stuff
+services:
+ wallabag_import.consumer.ampq.pocket:
+ class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.pocket.import"
+ - "@logger"
+ wallabag_import.consumer.ampq.readability:
+ class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.readability.import"
+ - "@logger"
+ wallabag_import.consumer.ampq.wallabag_v1:
+ class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.wallabag_v1.import"
+ - "@logger"
+ wallabag_import.consumer.ampq.wallabag_v2:
+ class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.wallabag_v2.import"
+ - "@logger"
--- /dev/null
+# Redis stuff
+services:
+ # readability
+ wallabag_import.queue.redis.readability:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ - "@wallabag_core.redis.client"
+ - "wallabag.import.readability"
+
+ wallabag_import.producer.redis.readability:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.readability"
+
+ wallabag_import.consumer.redis.readability:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.readability.import"
+ - "@logger"
+
+ # pocket
+ wallabag_import.queue.redis.pocket:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ - "@wallabag_core.redis.client"
+ - "wallabag.import.pocket"
+
+ wallabag_import.producer.redis.pocket:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.pocket"
+
+ wallabag_import.consumer.redis.pocket:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.pocket.import"
+ - "@logger"
+
+ # wallabag v1
+ wallabag_import.queue.redis.wallabag_v1:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ - "@wallabag_core.redis.client"
+ - "wallabag.import.wallabag_v1"
+
+ wallabag_import.producer.redis.wallabag_v1:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.wallabag_v1"
+
+ wallabag_import.consumer.redis.wallabag_v1:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.wallabag_v1.import"
+ - "@logger"
+
+ # wallabag v2
+ wallabag_import.queue.redis.wallabag_v2:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ - "@wallabag_core.redis.client"
+ - "wallabag.import.wallabag_v2"
+
+ wallabag_import.producer.redis.wallabag_v2:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.wallabag_v2"
+
+ wallabag_import.consumer.redis.wallabag_v2:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.wallabag_v2.import"
+ - "@logger"
-services:
- wallabag_import.consumer.pocket:
- class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
- arguments:
- - "@doctrine.orm.entity_manager"
- - "@wallabag_user.user_repository"
- - "@wallabag_import.pocket.import"
- - "@logger"
- wallabag_import.consumer.readability:
- class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
- arguments:
- - "@doctrine.orm.entity_manager"
- - "@wallabag_user.user_repository"
- - "@wallabag_import.readability.import"
- - "@logger"
- wallabag_import.consumer.wallabag_v1:
- class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
- arguments:
- - "@doctrine.orm.entity_manager"
- - "@wallabag_user.user_repository"
- - "@wallabag_import.wallabag_v1.import"
- - "@logger"
- wallabag_import.consumer.wallabag_v2:
- class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
- arguments:
- - "@doctrine.orm.entity_manager"
- - "@wallabag_user.user_repository"
- - "@wallabag_import.wallabag_v2.import"
- - "@logger"
+imports:
+ - { resource: rabbit.yml }
+ - { resource: redis.yml }
+services:
wallabag_import.chain:
class: Wallabag\ImportBundle\Import\ImportChain
namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
-use Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer;
+use Wallabag\ImportBundle\Consumer\AMPQEntryConsumer;
use PhpAmqpLib\Message\AMQPMessage;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
-class EntryConsumerTest extends \PHPUnit_Framework_TestCase
+class AMPQEntryConsumerTest extends \PHPUnit_Framework_TestCase
{
public function testMessageOk()
{
->with(json_decode($body, true))
->willReturn($entry);
- $consumer = new EntryConsumer(
+ $consumer = new AMPQEntryConsumer(
$em,
$userRepository,
$import
->disableOriginalConstructor()
->getMock();
- $consumer = new EntryConsumer(
+ $consumer = new AMPQEntryConsumer(
$em,
$userRepository,
$import
->with(json_decode($body, true))
->willReturn(null);
- $consumer = new EntryConsumer(
+ $consumer = new AMPQEntryConsumer(
$em,
$userRepository,
$import
--- /dev/null
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Consumer\AMQP;
+
+use Wallabag\ImportBundle\Consumer\RedisEntryConsumer;
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\CoreBundle\Entity\Entry;
+
+class RedisEntryConsumerTest extends \PHPUnit_Framework_TestCase
+{
+ public function testMessageOk()
+ {
+ $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $em
+ ->expects($this->once())
+ ->method('flush');
+
+ $em
+ ->expects($this->exactly(2))
+ ->method('clear');
+
+ $body = <<<'JSON'
+{
+ "item_id": "1402935436",
+ "resolved_id": "1402935436",
+ "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+ "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+ "favorite": "0",
+ "status": "0",
+ "time_added": "1473020899",
+ "time_updated": "1473020899",
+ "time_read": "0",
+ "time_favorited": "0",
+ "sort_id": 0,
+ "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules",
+ "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial",
+ "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...",
+ "is_article": "1",
+ "is_index": "0",
+ "has_video": "0",
+ "has_image": "1",
+ "word_count": "200",
+ "tags": {
+ "ifttt": {
+ "item_id": "1402935436",
+ "tag": "ifttt"
+ },
+ "mashable": {
+ "item_id": "1402935436",
+ "tag": "mashable"
+ }
+ },
+ "authors": {
+ "2484273": {
+ "item_id": "1402935436",
+ "author_id": "2484273",
+ "name": "Adam Rosenberg",
+ "url": "http://mashable.com/author/adam-rosenberg/"
+ }
+ },
+ "image": {
+ "item_id": "1402935436",
+ "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+ "width": "0",
+ "height": "0"
+ },
+ "images": {
+ "1": {
+ "item_id": "1402935436",
+ "image_id": "1",
+ "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg",
+ "width": "0",
+ "height": "0",
+ "credit": "Image: Steve Eichner/NameFace/Sipa USA",
+ "caption": ""
+ }
+ },
+ "userId": 1
+}
+JSON;
+
+ $user = new User();
+ $entry = new Entry($user);
+
+ $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $userRepository
+ ->expects($this->once())
+ ->method('find')
+ // userId from the body json above
+ ->with(1)
+ ->willReturn($user);
+
+ $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $import
+ ->expects($this->once())
+ ->method('setUser')
+ ->with($user);
+
+ $import
+ ->expects($this->once())
+ ->method('parseEntry')
+ ->with(json_decode($body, true))
+ ->willReturn($entry);
+
+ $consumer = new RedisEntryConsumer(
+ $em,
+ $userRepository,
+ $import
+ );
+
+ $res = $consumer->manage($body);
+
+ $this->assertTrue($res);
+ }
+
+ public function testMessageWithBadUser()
+ {
+ $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $em
+ ->expects($this->never())
+ ->method('flush');
+
+ $em
+ ->expects($this->never())
+ ->method('clear');
+
+ $body = '{ "userId": 123 }';
+
+ $user = new User();
+ $entry = new Entry($user);
+
+ $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $userRepository
+ ->expects($this->once())
+ ->method('find')
+ // userId from the body json above
+ ->with(123)
+ ->willReturn(null);
+
+ $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $consumer = new RedisEntryConsumer(
+ $em,
+ $userRepository,
+ $import
+ );
+
+ $res = $consumer->manage($body);
+
+ $this->assertFalse($res);
+ }
+
+ public function testMessageWithEntryProcessed()
+ {
+ $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $em
+ ->expects($this->never())
+ ->method('flush');
+
+ $em
+ ->expects($this->never())
+ ->method('clear');
+
+ $body = '{ "userId": 123 }';
+
+ $user = new User();
+
+ $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $userRepository
+ ->expects($this->once())
+ ->method('find')
+ // userId from the body json above
+ ->with(123)
+ ->willReturn($user);
+
+ $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $import
+ ->expects($this->once())
+ ->method('setUser')
+ ->with($user);
+
+ $import
+ ->expects($this->once())
+ ->method('parseEntry')
+ ->with(json_decode($body, true))
+ ->willReturn(null);
+
+ $consumer = new RedisEntryConsumer(
+ $em,
+ $userRepository,
+ $import
+ );
+
+ $res = $consumer->manage($body);
+
+ $this->assertFalse($res);
+ }
+}
$this->logInAs('admin');
$client = $this->getClient();
- $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/pocket');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
$this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
- $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+ }
+
+ public function testImportPocketWithRedisEnabled()
+ {
+ $this->logInAs('admin');
+ $client = $this->getClient();
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+ $crawler = $client->request('GET', '/import/pocket');
+
+ $this->assertEquals(200, $client->getResponse()->getStatusCode());
+ $this->assertEquals(1, $crawler->filter('button[type=submit]')->count());
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportPocketAuthBadToken()
$this->logInAs('admin');
$client = $this->getClient();
- $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/readability');
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
- $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+ }
+
+ public function testImportReadabilityWithRedisEnabled()
+ {
+ $this->logInAs('admin');
+ $client = $this->getClient();
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+ $crawler = $client->request('GET', '/import/readability');
+
+ $this->assertEquals(200, $client->getResponse()->getStatusCode());
+ $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportReadabilityWithFile()
$this->logInAs('admin');
$client = $this->getClient();
- $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/wallabag-v1');
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
- $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+ }
+
+ public function testImportWallabagWithRedisEnabled()
+ {
+ $this->logInAs('admin');
+ $client = $this->getClient();
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+ $crawler = $client->request('GET', '/import/wallabag-v1');
+
+ $this->assertEquals(200, $client->getResponse()->getStatusCode());
+ $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportWallabagWithFile()
$this->logInAs('admin');
$client = $this->getClient();
- $client->getContainer()->get('craue_config')->set('rabbitmq', 1);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1);
$crawler = $client->request('GET', '/import/wallabag-v2');
$this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
$this->assertEquals(1, $crawler->filter('input[type=file]')->count());
- $client->getContainer()->get('craue_config')->set('rabbitmq', 0);
+ $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0);
+ }
+
+ public function testImportWallabagWithRedisEnabled()
+ {
+ $this->logInAs('admin');
+ $client = $this->getClient();
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 1);
+
+ $crawler = $client->request('GET', '/import/wallabag-v2');
+
+ $this->assertEquals(200, $client->getResponse()->getStatusCode());
+ $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count());
+ $this->assertEquals(1, $crawler->filter('input[type=file]')->count());
+
+ $client->getContainer()->get('craue_config')->set('import_with_redis', 0);
}
public function testImportWallabagWithFile()
use GuzzleHttp\Subscriber\Mock;
use GuzzleHttp\Message\Response;
use GuzzleHttp\Stream\Stream;
+use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
class PocketImportTest extends \PHPUnit_Framework_TestCase
{
->with(json_encode($bodyAsArray));
$pocketImport->setClient($client);
- $pocketImport->setRabbitmqProducer($producer);
+ $pocketImport->setProducer($producer);
$pocketImport->authorize('wunderbar_code');
$res = $pocketImport->setMarkAsRead(true)->import();
$this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
}
+ /**
+ * Will sample results from https://getpocket.com/developer/docs/v3/retrieve.
+ */
+ public function testImportWithRedis()
+ {
+ $client = new Client();
+
+ $body = <<<'JSON'
+{
+ "item_id": "229279689",
+ "resolved_id": "229279689",
+ "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+ "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland",
+ "favorite": "1",
+ "status": "1",
+ "time_added": "1473020899",
+ "time_updated": "1473020899",
+ "time_read": "0",
+ "time_favorited": "0",
+ "sort_id": 0,
+ "resolved_title": "The Massive Ryder Cup Preview",
+ "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview",
+ "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.",
+ "is_article": "1",
+ "has_video": "0",
+ "has_image": "0",
+ "word_count": "3197"
+}
+JSON;
+
+ $mock = new Mock([
+ new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))),
+ new Response(200, ['Content-Type' => 'application/json'], Stream::factory('
+ {
+ "status": 1,
+ "list": {
+ "229279690": '.$body.'
+ }
+ }
+ ')),
+ ]);
+
+ $client->getEmitter()->attach($mock);
+
+ $pocketImport = $this->getPocketImport();
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = new Entry($this->user);
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter('Predis\Client', true);
+
+ $queue = new RedisQueue($redisMock, 'pocket');
+ $producer = new Producer($queue);
+
+ $pocketImport->setClient($client);
+ $pocketImport->setProducer($producer);
+ $pocketImport->authorize('wunderbar_code');
+
+ $res = $pocketImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('pocket'));
+ }
+
public function testImportBadResponse()
{
$client = new Client();
use Wallabag\ImportBundle\Import\ReadabilityImport;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
class ReadabilityImportTest extends \PHPUnit_Framework_TestCase
{
->expects($this->exactly(2))
->method('publish');
- $readabilityImport->setRabbitmqProducer($producer);
+ $readabilityImport->setProducer($producer);
$res = $readabilityImport->setMarkAsRead(true)->import();
$this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
}
+ public function testImportWithRedis()
+ {
+ $readabilityImport = $this->getReadabilityImport();
+ $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter('Predis\Client', true);
+
+ $queue = new RedisQueue($redisMock, 'readability');
+ $producer = new Producer($queue);
+
+ $readabilityImport->setProducer($producer);
+
+ $res = $readabilityImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('readability'));
+ }
+
public function testImportBadFile()
{
$readabilityImport = $this->getReadabilityImport();
use Wallabag\ImportBundle\Import\WallabagV1Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
{
->expects($this->exactly(4))
->method('publish');
- $wallabagV1Import->setRabbitmqProducer($producer);
+ $wallabagV1Import->setProducer($producer);
$res = $wallabagV1Import->setMarkAsRead(true)->import();
$this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
}
+ public function testImportWithRedis()
+ {
+ $wallabagV1Import = $this->getWallabagV1Import();
+ $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter('Predis\Client', true);
+
+ $queue = new RedisQueue($redisMock, 'wallabag_v1');
+ $producer = new Producer($queue);
+
+ $wallabagV1Import->setProducer($producer);
+
+ $res = $wallabagV1Import->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('wallabag_v1'));
+ }
+
public function testImportBadFile()
{
$wallabagV1Import = $this->getWallabagV1Import();
use Wallabag\ImportBundle\Import\WallabagV2Import;
use Wallabag\UserBundle\Entity\User;
use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
use Monolog\Logger;
use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase
{
->expects($this->exactly(24))
->method('publish');
- $wallabagV2Import->setRabbitmqProducer($producer);
+ $wallabagV2Import->setProducer($producer);
$res = $wallabagV2Import->setMarkAsRead(true)->import();
$this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
}
+ public function testImportWithRedis()
+ {
+ $wallabagV2Import = $this->getWallabagV2Import();
+ $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter('Predis\Client', true);
+
+ $queue = new RedisQueue($redisMock, 'wallabag_v2');
+ $producer = new Producer($queue);
+
+ $wallabagV2Import->setProducer($producer);
+
+ $res = $wallabagV2Import->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('wallabag_v2'));
+ }
+
public function testImportBadFile()
{
$wallabagV1Import = $this->getWallabagV2Import();
projects / github / wallabag / wallabag.git / commitdiff
