From 56c778b4152a1b886353933276ee3626e4e8c004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 15 Jan 2016 08:24:32 +0100 Subject: 1st draft for rabbitMQ --- app/AppKernel.php | 1 + app/config/config.yml | 25 +++++++++++ app/config/parameters.yml.dist | 12 ++++++ composer.json | 3 +- docs/en/developer/rabbitmq.rst | 49 ++++++++++++++++++++++ .../ImportBundle/Component/AMPQ/EntryConsumer.php | 39 +++++++++++++++++ src/Wallabag/ImportBundle/Import/PocketImport.php | 35 ++++++++++++---- .../ImportBundle/Resources/config/services.yml | 9 ++++ .../ImportBundle/Import/PocketImportTest.php | 11 ++++- 9 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 docs/en/developer/rabbitmq.rst create mode 100644 src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php diff --git a/app/AppKernel.php b/app/AppKernel.php index 96e45da8..52f85558 100644 --- a/app/AppKernel.php +++ b/app/AppKernel.php @@ -38,6 +38,7 @@ class AppKernel extends Kernel new Wallabag\UserBundle\WallabagUserBundle(), new Wallabag\ImportBundle\WallabagImportBundle(), new Wallabag\AnnotationBundle\WallabagAnnotationBundle(), + new OldSound\RabbitMqBundle\OldSoundRabbitMqBundle(), ]; if (in_array($this->getEnvironment(), ['dev', 'test'], true)) { diff --git a/app/config/config.yml b/app/config/config.yml index 31bd8a8c..ef5ae0aa 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -215,3 +215,28 @@ lexik_maintenance: response: code: 503 status: "wallabag Service Temporarily Unavailable" + +old_sound_rabbit_mq: + connections: + default: + host: %rabbitmq_host% + port: %rabbitmq_port% + user: %rabbitmq_user% + password: %rabbitmq_password% + vhost: / + lazy: false + producers: + wallabag: + connection: default + exchange_options: + name: 'wallabag_exchange' + type: topic + consumers: + entries: + connection: default + exchange_options: + name: 'wallabag_exchange' + type: topic + queue_options: + name: 'wallabag_queue' + callback: wallabag_import.consumer.entry diff --git a/app/config/parameters.yml.dist b/app/config/parameters.yml.dist index d092e139..e925b412 100644 --- a/app/config/parameters.yml.dist +++ b/app/config/parameters.yml.dist @@ -38,3 +38,15 @@ parameters: fosuser_confirmation: true from_email: no-reply@wallabag.org + + rss_limit: 50 + + # pocket import + pocket_consumer_key: xxxxxxxx + + # RabbitMQ processing + rabbitmq: false + rabbitmq_host: localhost + rabbitmq_port: 5672 + rabbitmq_user: guest + rabbitmq_password: guest diff --git a/composer.json b/composer.json index bdaad601..121a3fd3 100644 --- a/composer.json +++ b/composer.json @@ -81,7 +81,8 @@ "lexik/maintenance-bundle": "~2.1", "ocramius/proxy-manager": "1.*", "white-october/pagerfanta-bundle": "^1.0", - "mouf/nodejs-installer": "~1.0" + "mouf/nodejs-installer": "~1.0", + "php-amqplib/rabbitmq-bundle": "^1.8" }, "require-dev": { "doctrine/doctrine-fixtures-bundle": "~2.2", diff --git a/docs/en/developer/rabbitmq.rst b/docs/en/developer/rabbitmq.rst new file mode 100644 index 00000000..a17e6e4d --- /dev/null +++ b/docs/en/developer/rabbitmq.rst @@ -0,0 +1,49 @@ +Install RabbitMQ for asynchronous tasks +======================================= + +In order to launch asynchronous tasks (useful for huge imports for example), we use RabbitMQ. + +Requirements +------------ + +You need to have RabbitMQ installed on your server. + +Installation +~~~~~~~~~~~~ + +.. code:: bash + + wget https://www.rabbitmq.com/rabbitmq-signing-key-public.asc + apt-key add rabbitmq-signing-key-public.asc + apt-get update + apt-get install rabbitmq-server + +Configuration and launch +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + rabbitmq-plugins enable rabbitmq_management # (useful to have a web interface, available at http://localhost:15672/ (guest/guest) + rabbitmq-server -detached + +Stop RabbitMQ +~~~~~~~~~~~~~ + +.. code:: bash + + rabbitmqctl stop + + +Configure RabbitMQ in wallabag +------------------------------ + +Edit your ``parameters.yml`` file to edit RabbitMQ configuration. + +Launch RabbitMQ consumer +------------------------ + +Put this command in a cron job: + +.. code:: bash + + bin/console rabbitmq:consumer entries -w \ No newline at end of file diff --git a/src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php b/src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php new file mode 100644 index 00000000..7775f01c --- /dev/null +++ b/src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php @@ -0,0 +1,39 @@ +em = $em; + $this->entryRepository = $entryRepository; + $this->contentProxy = $contentProxy; + } + + /** + * {@inheritdoc} + */ + public function execute(AMQPMessage $msg) + { + $storedEntry = unserialize($msg->body); + $entry = $this->entryRepository->findByUrlAndUserId($storedEntry['url'], $storedEntry['userId']); + if ($entry) { + $entry = $this->contentProxy->updateEntry($entry, $entry->getUrl()); + if ($entry) { + $this->em->persist($entry); + $this->em->flush(); + } + } + } +} diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index a6f905b1..b02894f0 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,6 +2,8 @@ namespace Wallabag\ImportBundle\Import; +use OldSound\RabbitMqBundle\RabbitMq\Producer; +use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; @@ -20,14 +22,18 @@ class PocketImport extends AbstractImport private $importedEntries = 0; private $markAsRead; protected $accessToken; + private $producer; + private $rabbitMQ; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) + public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, $rabbitMQ, Producer $producer) { $this->user = $tokenStorage->getToken()->getUser(); $this->em = $em; $this->contentProxy = $contentProxy; $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); + $this->rabbitMQ = $rabbitMQ; + $this->producer = $producer; } /** @@ -197,7 +203,7 @@ class PocketImport extends AbstractImport { $i = 1; - foreach ($entries as $pocketEntry) { + foreach ($entries as &$pocketEntry) { $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; $existingEntry = $this->em @@ -210,12 +216,15 @@ class PocketImport extends AbstractImport } $entry = new Entry($this->user); - $entry = $this->fetchContent($entry, $url); - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; + if (!$this->rabbitMQ) { + $entry = $this->fetchContent($entry, $url); + + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; + continue; + } } // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted @@ -236,6 +245,7 @@ class PocketImport extends AbstractImport } $entry->setTitle($title); + $entry->setUrl($url); // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { @@ -249,6 +259,9 @@ class PocketImport extends AbstractImport ); } + $pocketEntry['url'] = $url; + $pocketEntry['userId'] = $this->user->getId(); + $this->em->persist($entry); ++$this->importedEntries; @@ -256,10 +269,16 @@ class PocketImport extends AbstractImport if (($i % 20) === 0) { $this->em->flush(); } + ++$i; } $this->em->flush(); - $this->em->clear(); + + if ($this->rabbitMQ) { + foreach ($entries as $entry) { + $this->producer->publish(serialize($entry)); + } + } } } diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 520d43af..7ea54162 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -1,4 +1,11 @@ services: + wallabag_import.consumer.entry: + class: Wallabag\ImportBundle\Component\AMPQ\EntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.entry_repository" + - "@wallabag_core.content_proxy" + wallabag_import.chain: class: Wallabag\ImportBundle\Import\ImportChain @@ -18,6 +25,8 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" - "@craue_config" + - %rabbitmq% + - "@old_sound_rabbit_mq.wallabag_producer" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 8534e1c8..a0f943ee 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -27,8 +27,9 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase protected $em; protected $contentProxy; protected $logHandler; + protected $producer; - private function getPocketImport($consumerKey = 'ConsumerKey') + private function getPocketImport($consumerKey = 'ConsumerKey', $rabbitMQ = false) { $this->user = new User(); @@ -65,11 +66,17 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase ->with('pocket_consumer_key') ->willReturn($consumerKey); + $this->producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + $pocket = new PocketImportMock( $this->tokenStorage, $this->em, $this->contentProxy, - $config + $config, + $rabbitMQ, + $this->producer ); $this->logHandler = new TestHandler(); -- cgit v1.2.3 From e31ee20dd7ed0f92e5c41c35a0876529d0b03436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Mon, 15 Feb 2016 20:35:28 +0100 Subject: Add RabbitMQ service for Travis CI --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 749b8ef7..918d15fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: php +services: + - rabbitmq + # faster builds on docker-container setup sudo: false -- cgit v1.2.3 From 40d2a29443df8ef6fdf1f2d09b5ba8808543c245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Mon, 15 Feb 2016 21:30:55 +0100 Subject: Replace RabbitMQ injection with CraueConfiguration --- .../CraueConfigBundle/translations/CraueConfigBundle.en.yml | 1 + src/Wallabag/CoreBundle/Command/InstallCommand.php | 5 +++++ src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php | 5 +++++ src/Wallabag/ImportBundle/Import/PocketImport.php | 4 ++-- src/Wallabag/ImportBundle/Resources/config/services.yml | 1 - tests/Wallabag/ImportBundle/Import/PocketImportTest.php | 1 - 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml index 48d0ec4b..731e6099 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml @@ -9,6 +9,7 @@ export_json: Enable JSON export export_txt: Enable TXT export export_xml: Enable XML export pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication) +rabbitmq: Enable RabbitMQ to import data asynchronously shaarli_url: Shaarli URL, if the service is enabled share_diaspora: Enable share to Diaspora share_mail: Enable share by email diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 143def4f..8a2439ec 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -321,6 +321,11 @@ class InstallCommand extends ContainerAwareCommand 'value' => null, 'section' => 'import', ], + [ + 'name' => 'rabbitmq', + 'value' => '0', + 'section' => 'import', + ], [ 'name' => 'show_printlink', 'value' => '1', diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index b4309304..778f91ed 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -95,6 +95,11 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface 'value' => null, 'section' => 'import', ], + [ + 'name' => 'rabbitmq', + 'value' => '0', + 'section' => 'import', + ], [ 'name' => 'show_printlink', 'value' => '1', diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index b02894f0..7d1c0c61 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -25,14 +25,14 @@ class PocketImport extends AbstractImport private $producer; private $rabbitMQ; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, $rabbitMQ, Producer $producer) + public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, Producer $producer) { $this->user = $tokenStorage->getToken()->getUser(); $this->em = $em; $this->contentProxy = $contentProxy; $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); - $this->rabbitMQ = $rabbitMQ; + $this->rabbitMQ = $craueConfig->get('rabbitmq'); $this->producer = $producer; } diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 7ea54162..60eb4e18 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -25,7 +25,6 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" - "@craue_config" - - %rabbitmq% - "@old_sound_rabbit_mq.wallabag_producer" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index a0f943ee..5bf47d96 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -75,7 +75,6 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $this->em, $this->contentProxy, $config, - $rabbitMQ, $this->producer ); -- cgit v1.2.3 From 87c9995b6c61a9f5cde3771bd4f9d44b5da26c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Tue, 23 Feb 2016 14:52:51 +0100 Subject: Update docker-composer with RabbitMQ configuration --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 50d8c546..a255b070 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,3 +46,7 @@ services: # - ./docker/data/mariadb:/var/lib/mysql # env_file: # - ./docker/mariadb/env + rabbitmq: + image: rabbitmq:3-management + ports: + - "15672:15672" -- cgit v1.2.3 From ef75e1220ebb76a8df019d946460ad612759f0bb Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 3 Sep 2016 17:36:57 +0200 Subject: Send every imported item to the queue Instead of queing real Entry to process, we queue all the item to import from Pocket in a raw format. Then, the worker retrieve that information, find / create the entry and save it. --- app/config/config.yml | 22 +-- app/config/parameters.yml.dist | 4 - .../ImportBundle/Component/AMPQ/EntryConsumer.php | 39 ----- .../ImportBundle/Consumer/AMPQ/PocketConsumer.php | 63 ++++++++ .../ImportBundle/Controller/PocketController.php | 25 ++- src/Wallabag/ImportBundle/Import/PocketImport.php | 177 +++++++++++++-------- .../ImportBundle/Resources/config/services.yml | 11 +- .../UserBundle/Resources/config/services.yml | 6 + .../ImportBundle/Import/PocketImportTest.php | 28 +--- 9 files changed, 221 insertions(+), 154 deletions(-) delete mode 100644 src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php create mode 100644 src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php diff --git a/app/config/config.yml b/app/config/config.yml index ef5ae0aa..fa829637 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -219,24 +219,24 @@ lexik_maintenance: old_sound_rabbit_mq: connections: default: - host: %rabbitmq_host% - port: %rabbitmq_port% - user: %rabbitmq_user% - password: %rabbitmq_password% + host: "%rabbitmq_host%" + port: "%rabbitmq_port%" + user: "%rabbitmq_user%" + password: "%rabbitmq_password%" vhost: / - lazy: false + lazy: true producers: - wallabag: + wallabag_pocket: connection: default exchange_options: - name: 'wallabag_exchange' + name: 'wallabag.import.pocket' type: topic consumers: - entries: + wallabag_pocket: connection: default exchange_options: - name: 'wallabag_exchange' + name: 'wallabag.import.pocket' type: topic queue_options: - name: 'wallabag_queue' - callback: wallabag_import.consumer.entry + name: 'wallabag.import.pocket' + callback: wallabag_import.consumer.pocket diff --git a/app/config/parameters.yml.dist b/app/config/parameters.yml.dist index e925b412..a59dc02c 100644 --- a/app/config/parameters.yml.dist +++ b/app/config/parameters.yml.dist @@ -41,11 +41,7 @@ parameters: rss_limit: 50 - # pocket import - pocket_consumer_key: xxxxxxxx - # RabbitMQ processing - rabbitmq: false rabbitmq_host: localhost rabbitmq_port: 5672 rabbitmq_user: guest diff --git a/src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php b/src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php deleted file mode 100644 index 7775f01c..00000000 --- a/src/Wallabag/ImportBundle/Component/AMPQ/EntryConsumer.php +++ /dev/null @@ -1,39 +0,0 @@ -em = $em; - $this->entryRepository = $entryRepository; - $this->contentProxy = $contentProxy; - } - - /** - * {@inheritdoc} - */ - public function execute(AMQPMessage $msg) - { - $storedEntry = unserialize($msg->body); - $entry = $this->entryRepository->findByUrlAndUserId($storedEntry['url'], $storedEntry['userId']); - if ($entry) { - $entry = $this->contentProxy->updateEntry($entry, $entry->getUrl()); - if ($entry) { - $this->em->persist($entry); - $this->em->flush(); - } - } - } -} diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php new file mode 100644 index 00000000..239e7446 --- /dev/null +++ b/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php @@ -0,0 +1,63 @@ +em = $em; + $this->userRepository = $userRepository; + $this->pocketImport = $pocketImport; + $this->logger = $logger ?: new NullLogger(); + } + + /** + * {@inheritdoc} + */ + public function execute(AMQPMessage $msg) + { + $storedEntry = json_decode($msg->body, true); + + $user = $this->userRepository->find($storedEntry['userId']); + + // no user? Drop message + if (null === $user) { + $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); + + return; + } + + $this->pocketImport->setUser($user); + + $entry = $this->pocketImport->parseEntry($storedEntry); + + if (null === $entry) { + $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); + + return; + } + + try { + $this->em->flush(); + $this->em->clear($entry); + } catch (\Exception $e) { + $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); + + return; + } + } +} diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 36ee25bf..a2dcd8a7 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -10,12 +10,29 @@ use Symfony\Component\Form\Extension\Core\Type\CheckboxType; class PocketController extends Controller { + /** + * Return Pocket Import Service with or without RabbitMQ enabled. + * + * @return \Wallabag\ImportBundle\Import\PocketImport + */ + private function getPocketImportService() + { + $pocket = $this->get('wallabag_import.pocket.import'); + $pocket->setUser($this->getUser()); + + if ($this->get('craue_config')->get('rabbitmq')) { + $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.wallabag_pocket_producer')); + } + + return $pocket; + } + /** * @Route("/pocket", name="import_pocket") */ public function indexAction() { - $pocket = $this->get('wallabag_import.pocket.import'); + $pocket = $this->getPocketImportService(); $form = $this->createFormBuilder($pocket) ->add('mark_as_read', CheckboxType::class, [ 'label' => 'import.form.mark_as_read_label', @@ -24,7 +41,7 @@ class PocketController extends Controller ->getForm(); return $this->render('WallabagImportBundle:Pocket:index.html.twig', [ - 'import' => $this->get('wallabag_import.pocket.import'), + 'import' => $this->getPocketImportService(), 'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true, 'form' => $form->createView(), ]); @@ -35,7 +52,7 @@ class PocketController extends Controller */ public function authAction(Request $request) { - $requestToken = $this->get('wallabag_import.pocket.import') + $requestToken = $this->getPocketImportService() ->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL)); if (false === $requestToken) { @@ -62,7 +79,7 @@ class PocketController extends Controller public function callbackAction() { $message = 'flashes.import.notice.failed'; - $pocket = $this->get('wallabag_import.pocket.import'); + $pocket = $this->getPocketImportService(); $markAsRead = $this->get('session')->get('mark_as_read'); $this->get('session')->remove('mark_as_read'); diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 7d1c0c61..27df4917 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -3,12 +3,11 @@ namespace Wallabag\ImportBundle\Import; use OldSound\RabbitMqBundle\RabbitMq\Producer; -use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; -use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; +use Symfony\Component\Security\Core\User\UserInterface; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Helper\ContentProxy; use Craue\ConfigBundle\Util\Config; @@ -21,21 +20,39 @@ class PocketImport extends AbstractImport private $skippedEntries = 0; private $importedEntries = 0; private $markAsRead; - protected $accessToken; private $producer; - private $rabbitMQ; + protected $accessToken; - public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig, Producer $producer) + public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) { - $this->user = $tokenStorage->getToken()->getUser(); $this->em = $em; $this->contentProxy = $contentProxy; $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); - $this->rabbitMQ = $craueConfig->get('rabbitmq'); + } + + /** + * Set RabbitMQ Producer to send each entry to a queue. + * This method should be called when user has enabled RabbitMQ. + * + * @param Producer $producer + */ + public function setRabbitmqProducer(Producer $producer) + { $this->producer = $producer; } + /** + * Set current user. + * Could the current *connected* user or one retrieve by the consumer. + * + * @param UserInterface $user + */ + public function setUser(UserInterface $user) + { + $this->user = $user; + } + /** * {@inheritdoc} */ @@ -168,6 +185,12 @@ class PocketImport extends AbstractImport $entries = $response->json(); + if ($this->producer) { + $this->parseEntriesForProducer($entries['list']); + + return true; + } + $this->parseEntries($entries['list']); return true; @@ -197,88 +220,112 @@ class PocketImport extends AbstractImport /** * @see https://getpocket.com/developer/docs/v3/retrieve * - * @param $entries + * @param array $entries */ - private function parseEntries($entries) + private function parseEntries(array $entries) { $i = 1; - foreach ($entries as &$pocketEntry) { - $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($url, $this->user->getId()); + foreach ($entries as $pocketEntry) { + $entry = $this->parseEntry($pocketEntry); - if (false !== $existingEntry) { - ++$this->skippedEntries; + if (null === $entry) { continue; } - $entry = new Entry($this->user); + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } - if (!$this->rabbitMQ) { - $entry = $this->fetchContent($entry, $url); + ++$i; + } - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; - } - } + $this->em->flush(); + } - // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($pocketEntry['status'] == 1 || $this->markAsRead) { - $entry->setArchived(true); - } + public function parseEntry(array $pocketEntry) + { + $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; - // 0 or 1 - 1 If the item is starred - if ($pocketEntry['favorite'] == 1) { - $entry->setStarred(true); - } + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->user->getId()); - $title = 'Untitled'; - if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { - $title = $pocketEntry['resolved_title']; - } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { - $title = $pocketEntry['given_title']; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - $entry->setTitle($title); - $entry->setUrl($url); + return; + } - // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image - if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { - $entry->setPreviewPicture($pocketEntry['images'][1]['src']); - } + $entry = new Entry($this->user); + $entry = $this->fetchContent($entry, $url); - if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { - $this->contentProxy->assignTagsToEntry( - $entry, - array_keys($pocketEntry['tags']) - ); - } + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; - $pocketEntry['url'] = $url; - $pocketEntry['userId'] = $this->user->getId(); + return; + } - $this->em->persist($entry); - ++$this->importedEntries; + // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted + if ($pocketEntry['status'] == 1 || $this->markAsRead) { + $entry->setArchived(true); + } - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } + // 0 or 1 - 1 If the item is starred + if ($pocketEntry['favorite'] == 1) { + $entry->setStarred(true); + } - ++$i; + $title = 'Untitled'; + if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { + $title = $pocketEntry['resolved_title']; + } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { + $title = $pocketEntry['given_title']; } - $this->em->flush(); + $entry->setTitle($title); + $entry->setUrl($url); + + // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image + if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { + $entry->setPreviewPicture($pocketEntry['images'][1]['src']); + } + + if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { + $this->contentProxy->assignTagsToEntry( + $entry, + array_keys($pocketEntry['tags']) + ); + } + + $this->em->persist($entry); + ++$this->importedEntries; - if ($this->rabbitMQ) { - foreach ($entries as $entry) { - $this->producer->publish(serialize($entry)); + return $entry; + } + + /** + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + public function parseEntriesForProducer($entries) + { + foreach ($entries as $pocketEntry) { + // set userId for the producer (it won't know which user is connected) + $pocketEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $pocketEntry['status'] = 1; } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($pocketEntry)); } } } diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 60eb4e18..fe388b26 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -1,10 +1,11 @@ services: - wallabag_import.consumer.entry: - class: Wallabag\ImportBundle\Component\AMPQ\EntryConsumer + wallabag_import.consumer.pocket: + class: Wallabag\ImportBundle\Consumer\AMPQ\PocketConsumer arguments: - "@doctrine.orm.entity_manager" - - "@wallabag_core.entry_repository" - - "@wallabag_core.content_proxy" + - "@wallabag_user.user_repository" + - "@wallabag_import.pocket.import" + - "@logger" wallabag_import.chain: class: Wallabag\ImportBundle\Import\ImportChain @@ -21,11 +22,9 @@ services: wallabag_import.pocket.import: class: Wallabag\ImportBundle\Import\PocketImport arguments: - - "@security.token_storage" - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" - "@craue_config" - - "@old_sound_rabbit_mq.wallabag_producer" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] diff --git a/src/Wallabag/UserBundle/Resources/config/services.yml b/src/Wallabag/UserBundle/Resources/config/services.yml index d79d8fa2..05830555 100644 --- a/src/Wallabag/UserBundle/Resources/config/services.yml +++ b/src/Wallabag/UserBundle/Resources/config/services.yml @@ -14,3 +14,9 @@ services: - "@router" tags: - { name: kernel.event_subscriber } + + wallabag_user.user_repository: + class: Wallabag\UserBundle\Repository\UserRepository + factory: [ "@doctrine.orm.default_entity_manager", getRepository ] + arguments: + - WallabagUserBundle:User diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 5bf47d96..d6b9617e 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -27,32 +27,15 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase protected $em; protected $contentProxy; protected $logHandler; - protected $producer; - private function getPocketImport($consumerKey = 'ConsumerKey', $rabbitMQ = false) + private function getPocketImport($consumerKey = 'ConsumerKey') { $this->user = new User(); - $this->tokenStorage = $this->getMockBuilder('Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface') - ->disableOriginalConstructor() - ->getMock(); - - $token = $this->getMockBuilder('Symfony\Component\Security\Core\Authentication\Token\TokenInterface') - ->disableOriginalConstructor() - ->getMock(); - $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') ->disableOriginalConstructor() ->getMock(); - $token->expects($this->once()) - ->method('getUser') - ->willReturn($this->user); - - $this->tokenStorage->expects($this->once()) - ->method('getToken') - ->willReturn($token); - $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') ->disableOriginalConstructor() ->getMock(); @@ -66,17 +49,12 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase ->with('pocket_consumer_key') ->willReturn($consumerKey); - $this->producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') - ->disableOriginalConstructor() - ->getMock(); - $pocket = new PocketImportMock( - $this->tokenStorage, $this->em, $this->contentProxy, - $config, - $this->producer + $config ); + $pocket->setUser($this->user); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); -- cgit v1.2.3 From c98db1b653b5dc8b701422190b02d9fbf10c4e68 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 4 Sep 2016 21:49:21 +0200 Subject: Convert other imports to Rabbit --- app/config/config.yml | 43 ++++++- .../ImportBundle/Consumer/AMPQ/EntryConsumer.php | 63 ++++++++++ .../ImportBundle/Consumer/AMPQ/PocketConsumer.php | 63 ---------- .../ImportBundle/Controller/PocketController.php | 2 +- .../Controller/ReadabilityController.php | 6 +- .../ImportBundle/Controller/WallabagController.php | 2 +- .../Controller/WallabagV1Controller.php | 8 +- .../Controller/WallabagV2Controller.php | 8 +- .../ImportBundle/Import/AbstractImport.php | 83 ++++++++++++++ src/Wallabag/ImportBundle/Import/PocketImport.php | 107 +++-------------- .../ImportBundle/Import/ReadabilityImport.php | 127 +++++++++------------ .../ImportBundle/Import/WallabagImport.php | 118 +++++++------------ .../ImportBundle/Import/WallabagV1Import.php | 20 +++- .../ImportBundle/Import/WallabagV2Import.php | 20 +++- .../ImportBundle/Resources/config/services.yml | 23 +++- 15 files changed, 382 insertions(+), 311 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php delete mode 100644 src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php diff --git a/app/config/config.yml b/app/config/config.yml index fa829637..d39bef95 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -226,13 +226,28 @@ old_sound_rabbit_mq: vhost: / lazy: true producers: - wallabag_pocket: + import_pocket: connection: default exchange_options: name: 'wallabag.import.pocket' type: topic + import_readability: + connection: default + exchange_options: + name: 'wallabag.import.readability' + type: topic + import_wallabag_v1: + connection: default + exchange_options: + name: 'wallabag.import.wallabag_v1' + type: topic + import_wallabag_v2: + connection: default + exchange_options: + name: 'wallabag.import.wallabag_v2' + type: topic consumers: - wallabag_pocket: + import_pocket: connection: default exchange_options: name: 'wallabag.import.pocket' @@ -240,3 +255,27 @@ old_sound_rabbit_mq: queue_options: name: 'wallabag.import.pocket' callback: wallabag_import.consumer.pocket + import_readability: + connection: default + exchange_options: + name: 'wallabag.import.readability' + type: topic + queue_options: + name: 'wallabag.import.readability' + callback: wallabag_import.consumer.readability + import_wallabag_v1: + connection: default + exchange_options: + name: 'wallabag.import.wallabag_v1' + type: topic + queue_options: + name: 'wallabag.import.wallabag_v1' + callback: wallabag_import.consumer.wallabag_v1 + import_wallabag_v2: + connection: default + exchange_options: + name: 'wallabag.import.wallabag_v2' + type: topic + queue_options: + name: 'wallabag.import.wallabag_v2' + callback: wallabag_import.consumer.wallabag_v2 diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php new file mode 100644 index 00000000..8a8cf45d --- /dev/null +++ b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php @@ -0,0 +1,63 @@ +em = $em; + $this->userRepository = $userRepository; + $this->import = $import; + $this->logger = $logger ?: new NullLogger(); + } + + /** + * {@inheritdoc} + */ + public function execute(AMQPMessage $msg) + { + $storedEntry = json_decode($msg->body, true); + + $user = $this->userRepository->find($storedEntry['userId']); + + // no user? Drop message + if (null === $user) { + $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); + + return; + } + + $this->import->setUser($user); + + $entry = $this->import->parseEntry($storedEntry); + + if (null === $entry) { + $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); + + return; + } + + try { + $this->em->flush(); + $this->em->clear($entry); + } catch (\Exception $e) { + $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); + + return; + } + } +} diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php deleted file mode 100644 index 239e7446..00000000 --- a/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php +++ /dev/null @@ -1,63 +0,0 @@ -em = $em; - $this->userRepository = $userRepository; - $this->pocketImport = $pocketImport; - $this->logger = $logger ?: new NullLogger(); - } - - /** - * {@inheritdoc} - */ - public function execute(AMQPMessage $msg) - { - $storedEntry = json_decode($msg->body, true); - - $user = $this->userRepository->find($storedEntry['userId']); - - // no user? Drop message - if (null === $user) { - $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); - - return; - } - - $this->pocketImport->setUser($user); - - $entry = $this->pocketImport->parseEntry($storedEntry); - - if (null === $entry) { - $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); - - return; - } - - try { - $this->em->flush(); - $this->em->clear($entry); - } catch (\Exception $e) { - $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); - - return; - } - } -} diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index a2dcd8a7..57c007c3 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -21,7 +21,7 @@ class PocketController extends Controller $pocket->setUser($this->getUser()); if ($this->get('craue_config')->get('rabbitmq')) { - $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.wallabag_pocket_producer')); + $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); } return $pocket; diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index b61aa99c..ee875a40 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -18,6 +18,11 @@ class ReadabilityController extends Controller $form->handleRequest($request); $readability = $this->get('wallabag_import.readability.import'); + $readability->setUser($this->getUser()); + + if ($this->get('craue_config')->get('rabbitmq')) { + $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); + } if ($form->isValid()) { $file = $form->get('file')->getData(); @@ -26,7 +31,6 @@ class ReadabilityController extends Controller if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $readability - ->setUser($this->getUser()) ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) ->import(); diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php index 76ced0d2..1e6114c5 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagController.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php @@ -38,6 +38,7 @@ abstract class WallabagController extends Controller $form->handleRequest($request); $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); if ($form->isValid()) { $file = $form->get('file')->getData(); @@ -46,7 +47,6 @@ abstract class WallabagController extends Controller if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $wallabag - ->setUser($this->getUser()) ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) ->import(); diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php index 3e748d57..f80aec3a 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php @@ -12,7 +12,13 @@ class WallabagV1Controller extends WallabagController */ protected function getImportService() { - return $this->get('wallabag_import.wallabag_v1.import'); + $service = $this->get('wallabag_import.wallabag_v1.import'); + + if ($this->get('craue_config')->get('rabbitmq')) { + $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } + + return $service; } /** diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php index c2a42165..063cddd9 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php @@ -12,7 +12,13 @@ class WallabagV2Controller extends WallabagController */ protected function getImportService() { - return $this->get('wallabag_import.wallabag_v2.import'); + $service = $this->get('wallabag_import.wallabag_v2.import'); + + if ($this->get('craue_config')->get('rabbitmq')) { + $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); + } + + return $service; } /** diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 14377a35..5b9d65d7 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -7,12 +7,17 @@ use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; +use Symfony\Component\Security\Core\User\UserInterface; +use OldSound\RabbitMqBundle\RabbitMq\Producer; abstract class AbstractImport implements ImportInterface { protected $em; protected $logger; protected $contentProxy; + protected $producer; + protected $user; + protected $markAsRead; public function __construct(EntityManager $em, ContentProxy $contentProxy) { @@ -26,6 +31,48 @@ abstract class AbstractImport implements ImportInterface $this->logger = $logger; } + /** + * Set RabbitMQ Producer to send each entry to a queue. + * This method should be called when user has enabled RabbitMQ. + * + * @param Producer $producer + */ + public function setRabbitmqProducer(Producer $producer) + { + $this->producer = $producer; + } + + /** + * Set current user. + * Could the current *connected* user or one retrieve by the consumer. + * + * @param UserInterface $user + */ + public function setUser(UserInterface $user) + { + $this->user = $user; + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Get whether articles must be all marked as read. + */ + public function getMarkAsRead() + { + return $this->markAsRead; + } + /** * Fetch content from the ContentProxy (using graby). * If it fails return false instead of the updated entry. @@ -44,4 +91,40 @@ abstract class AbstractImport implements ImportInterface return false; } } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) + { + $i = 1; + + foreach ($entries as $importedEntry) { + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$i; + } + + $this->em->flush(); + } + + /** + * Parse one entry. + * + * @param array $importedEntry + * + * @return Entry + */ + abstract public function parseEntry(array $importedEntry); } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 27df4917..dd0ddd72 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,25 +2,20 @@ namespace Wallabag\ImportBundle\Import; -use OldSound\RabbitMqBundle\RabbitMq\Producer; use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; -use Symfony\Component\Security\Core\User\UserInterface; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Helper\ContentProxy; use Craue\ConfigBundle\Util\Config; class PocketImport extends AbstractImport { - private $user; private $client; private $consumerKey; private $skippedEntries = 0; private $importedEntries = 0; - private $markAsRead; - private $producer; protected $accessToken; public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) @@ -31,28 +26,6 @@ class PocketImport extends AbstractImport $this->logger = new NullLogger(); } - /** - * Set RabbitMQ Producer to send each entry to a queue. - * This method should be called when user has enabled RabbitMQ. - * - * @param Producer $producer - */ - public function setRabbitmqProducer(Producer $producer) - { - $this->producer = $producer; - } - - /** - * Set current user. - * Could the current *connected* user or one retrieve by the consumer. - * - * @param UserInterface $user - */ - public function setUser(UserInterface $user) - { - $this->user = $user; - } - /** * {@inheritdoc} */ @@ -138,26 +111,6 @@ class PocketImport extends AbstractImport return true; } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; - - return $this; - } - - /** - * Get whether articles must be all marked as read. - */ - public function getMarkAsRead() - { - return $this->markAsRead; - } - /** * {@inheritdoc} */ @@ -217,37 +170,9 @@ class PocketImport extends AbstractImport $this->client = $client; } - /** - * @see https://getpocket.com/developer/docs/v3/retrieve - * - * @param array $entries - */ - private function parseEntries(array $entries) - { - $i = 1; - - foreach ($entries as $pocketEntry) { - $entry = $this->parseEntry($pocketEntry); - - if (null === $entry) { - continue; - } - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - $this->em->clear($entry); - } - - ++$i; - } - - $this->em->flush(); - } - - public function parseEntry(array $pocketEntry) + public function parseEntry(array $importedEntry) { - $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; + $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; $existingEntry = $this->em ->getRepository('WallabagCoreBundle:Entry') @@ -270,34 +195,34 @@ class PocketImport extends AbstractImport } // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($pocketEntry['status'] == 1 || $this->markAsRead) { + if ($importedEntry['status'] == 1 || $this->markAsRead) { $entry->setArchived(true); } // 0 or 1 - 1 If the item is starred - if ($pocketEntry['favorite'] == 1) { + if ($importedEntry['favorite'] == 1) { $entry->setStarred(true); } $title = 'Untitled'; - if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { - $title = $pocketEntry['resolved_title']; - } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { - $title = $pocketEntry['given_title']; + if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') { + $title = $importedEntry['resolved_title']; + } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') { + $title = $importedEntry['given_title']; } $entry->setTitle($title); $entry->setUrl($url); // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image - if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { - $entry->setPreviewPicture($pocketEntry['images'][1]['src']); + if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { + $entry->setPreviewPicture($importedEntry['images'][1]['src']); } - if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { + if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { $this->contentProxy->assignTagsToEntry( $entry, - array_keys($pocketEntry['tags']) + array_keys($importedEntry['tags']) ); } @@ -315,17 +240,17 @@ class PocketImport extends AbstractImport */ public function parseEntriesForProducer($entries) { - foreach ($entries as $pocketEntry) { + foreach ($entries as $importedEntry) { // set userId for the producer (it won't know which user is connected) - $pocketEntry['userId'] = $this->user->getId(); + $importedEntry['userId'] = $this->user->getId(); if ($this->markAsRead) { - $pocketEntry['status'] = 1; + $importedEntry['status'] = 1; } ++$this->importedEntries; - $this->producer->publish(json_encode($pocketEntry)); + $this->producer->publish(json_encode($importedEntry)); } } } diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index c7cfe15d..18a6631a 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -7,24 +7,9 @@ use Wallabag\UserBundle\Entity\User; class ReadabilityImport extends AbstractImport { - private $user; private $skippedEntries = 0; private $importedEntries = 0; private $filepath; - private $markAsRead; - - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } /** * {@inheritdoc} @@ -62,26 +47,6 @@ class ReadabilityImport extends AbstractImport return $this; } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; - - return $this; - } - - /** - * Get whether articles must be all marked as read. - */ - public function getMarkAsRead() - { - return $this->markAsRead; - } - /** * {@inheritdoc} */ @@ -116,54 +81,76 @@ class ReadabilityImport extends AbstractImport return false; } + if ($this->producer) { + $this->parseEntriesForProducer($data['bookmarks']); + + return true; + } + $this->parseEntries($data['bookmarks']); return true; } + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $data = [ + 'title' => $importedEntry['article__title'], + 'url' => $importedEntry['article__url'], + 'content_type' => '', + 'language' => '', + 'is_archived' => $importedEntry['archive'] || $this->markAsRead, + 'is_starred' => $importedEntry['favorite'], + ]; + + $entry = $this->fetchContent( + new Entry($this->user), + $data['url'], + $data + ); + + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; + + return; + } + + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + /** - * Parse and insert all given entries. + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. * - * @param $entries + * @param array $entries */ - protected function parseEntries($entries) + protected function parseEntriesForProducer($entries) { - $i = 1; - foreach ($entries as $importedEntry) { - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); - $data = [ - 'title' => $importedEntry['article__title'], - 'url' => $importedEntry['article__url'], - 'content_type' => '', - 'language' => '', - 'is_archived' => $importedEntry['archive'] || $this->markAsRead, - 'is_starred' => $importedEntry['favorite'], - ]; - - $entry = $this->fetchContent( - new Entry($this->user), - $data['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; + if ($this->markAsRead) { + $importedEntry['archive'] = 1; } - $entry->setArchived($data['is_archived']); - $entry->setStarred($data['is_starred']); - $this->em->persist($entry); ++$this->importedEntries; // flush every 20 entries diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 581ec178..6ad14e8c 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -11,7 +11,6 @@ abstract class WallabagImport extends AbstractImport protected $skippedEntries = 0; protected $importedEntries = 0; protected $filepath; - protected $markAsRead; // untitled in all languages from v1 protected $untitled = [ 'Untitled', @@ -28,19 +27,6 @@ abstract class WallabagImport extends AbstractImport '', ]; - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } - /** * {@inheritdoc} */ @@ -79,6 +65,12 @@ abstract class WallabagImport extends AbstractImport return false; } + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); return true; @@ -108,85 +100,61 @@ abstract class WallabagImport extends AbstractImport } /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead + * {@inheritdoc} */ - public function setMarkAsRead($markAsRead) + public function parseEntry(array $importedEntry) { - $this->markAsRead = $markAsRead; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); - return $this; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - /** - * Parse and insert all given entries. - * - * @param $entries - */ - protected function parseEntries($entries) - { - $i = 1; + return; + } + + $data = $this->prepareEntry($importedEntry); - foreach ($entries as $importedEntry) { - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + $entry = $this->fetchContent( + new Entry($this->user), + $importedEntry['url'], + $data + ); - if (false !== $existingEntry) { - ++$this->skippedEntries; - continue; - } + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; - $data = $this->prepareEntry($importedEntry, $this->markAsRead); + return; + } - $entry = $this->fetchContent( - new Entry($this->user), - $importedEntry['url'], - $data + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] ); + } - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; - continue; - } - - if (array_key_exists('tags', $data)) { - $this->contentProxy->assignTagsToEntry( - $entry, - $data['tags'] - ); - } - - if (isset($importedEntry['preview_picture'])) { - $entry->setPreviewPicture($importedEntry['preview_picture']); - } - - $entry->setArchived($data['is_archived']); - $entry->setStarred($data['is_starred']); - - $this->em->persist($entry); - ++$this->importedEntries; - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } - ++$i; + if (isset($importedEntry['preview_picture'])) { + $entry->setPreviewPicture($importedEntry['preview_picture']); } - $this->em->flush(); - $this->em->clear(); + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; } /** * This should return a cleaned array for a given entry to be given to `updateEntry`. * - * @param array $entry Data from the imported file - * @param bool $markAsRead Should we mark as read content? + * @param array $entry Data from the imported file * * @return array */ - abstract protected function prepareEntry($entry = [], $markAsRead = false); + abstract protected function prepareEntry($entry = []); } diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 6cf3467a..86734652 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport /** * {@inheritdoc} */ - protected function prepareEntry($entry = [], $markAsRead = false) + protected function prepareEntry($entry = []) { $data = [ 'title' => $entry['title'], @@ -39,7 +39,7 @@ class WallabagV1Import extends WallabagImport 'url' => $entry['url'], 'content_type' => '', 'language' => '', - 'is_archived' => $entry['is_read'] || $markAsRead, + 'is_archived' => $entry['is_read'] || $this->markAsRead, 'is_starred' => $entry['is_fav'], 'tags' => '', ]; @@ -56,4 +56,20 @@ class WallabagV1Import extends WallabagImport return $data; } + + protected function parseEntriesForProducer($entries) + { + foreach ($entries as $importedEntry) { + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry['is_read'] = 1; + } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php index d0035b63..faf4236f 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php @@ -31,12 +31,28 @@ class WallabagV2Import extends WallabagImport /** * {@inheritdoc} */ - protected function prepareEntry($entry = [], $markAsRead = false) + protected function prepareEntry($entry = []) { return [ 'html' => $entry['content'], 'content_type' => $entry['mimetype'], - 'is_archived' => ($entry['is_archived'] || $markAsRead), + 'is_archived' => ($entry['is_archived'] || $this->markAsRead), ] + $entry; } + + protected function parseEntriesForProducer($entries) + { + foreach ($entries as $importedEntry) { + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry['is_archived'] = 1; + } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } } diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index fe388b26..cad44e71 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -1,11 +1,32 @@ services: wallabag_import.consumer.pocket: - class: Wallabag\ImportBundle\Consumer\AMPQ\PocketConsumer + class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer arguments: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" - "@logger" + wallabag_import.consumer.readability: + class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.readability.import" + - "@logger" + wallabag_import.consumer.wallabag_v1: + class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.wallabag_v1.import" + - "@logger" + wallabag_import.consumer.wallabag_v2: + class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.wallabag_v2.import" + - "@logger" wallabag_import.chain: class: Wallabag\ImportBundle\Import\ImportChain -- cgit v1.2.3 From 02f64895728fe9aee2c696a627e0bbe27a24faf2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 07:13:09 +0200 Subject: Retrieve all items from Pocket 5000 by 5000. Also, retrieve newest item first. --- src/Wallabag/ImportBundle/Import/PocketImport.php | 37 ++++++++++++++++++---- .../ImportBundle/Import/PocketImportTest.php | 10 +----- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index dd0ddd72..06a31813 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -16,7 +16,9 @@ class PocketImport extends AbstractImport private $consumerKey; private $skippedEntries = 0; private $importedEntries = 0; - protected $accessToken; + private $accessToken; + + const NB_ELEMENTS = 5000; public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) { @@ -26,6 +28,16 @@ class PocketImport extends AbstractImport $this->logger = new NullLogger(); } + /** + * Only used for test purpose + * + * @return string + */ + public function getAccessToken() + { + return $this->accessToken; + } + /** * {@inheritdoc} */ @@ -114,8 +126,10 @@ class PocketImport extends AbstractImport /** * {@inheritdoc} */ - public function import() + public function import($offset = 0) { + static $run = 0; + $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', [ 'body' => json_encode([ @@ -123,7 +137,9 @@ class PocketImport extends AbstractImport 'access_token' => $this->accessToken, 'detailType' => 'complete', 'state' => 'all', - 'sort' => 'oldest', + 'sort' => 'newest', + 'count' => self::NB_ELEMENTS, + 'offset' => $offset, ]), ] ); @@ -140,11 +156,20 @@ class PocketImport extends AbstractImport if ($this->producer) { $this->parseEntriesForProducer($entries['list']); - - return true; + } else { + $this->parseEntries($entries['list']); } - $this->parseEntries($entries['list']); + // if we retrieve exactly the amount of items requested it means we can get more + // re-call import and offset item by the amount previous received: + // - first call get 5k offset 0 + // - second call get 5k offset 5k + // - and so on + if (count($entries['list']) === self::NB_ELEMENTS) { + ++$run; + + return $this->import(self::NB_ELEMENTS * $run); + } return true; } diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index d6b9617e..26c6b825 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -12,14 +12,6 @@ use GuzzleHttp\Stream\Stream; use Monolog\Logger; use Monolog\Handler\TestHandler; -class PocketImportMock extends PocketImport -{ - public function getAccessToken() - { - return $this->accessToken; - } -} - class PocketImportTest extends \PHPUnit_Framework_TestCase { protected $token; @@ -49,7 +41,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase ->with('pocket_consumer_key') ->willReturn($consumerKey); - $pocket = new PocketImportMock( + $pocket = new PocketImport( $this->em, $this->contentProxy, $config -- cgit v1.2.3 From 3849a9f3231c0109c87af085452c3ac5e4aed303 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 07:50:10 +0200 Subject: Some cleanup & refactor --- .../ImportBundle/Import/AbstractImport.php | 42 ++++++++++++++++++++-- src/Wallabag/ImportBundle/Import/PocketImport.php | 22 +++--------- .../ImportBundle/Import/ReadabilityImport.php | 28 +++------------ .../ImportBundle/Import/WallabagImport.php | 2 -- .../ImportBundle/Import/WallabagV1Import.php | 18 ++++------ .../ImportBundle/Import/WallabagV2Import.php | 18 ++++------ 6 files changed, 60 insertions(+), 70 deletions(-) diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 5b9d65d7..b085dc3a 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -7,7 +7,7 @@ use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; -use Symfony\Component\Security\Core\User\UserInterface; +use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\Producer; abstract class AbstractImport implements ImportInterface @@ -46,9 +46,9 @@ abstract class AbstractImport implements ImportInterface * Set current user. * Could the current *connected* user or one retrieve by the consumer. * - * @param UserInterface $user + * @param User $user */ - public function setUser(UserInterface $user) + public function setUser(User $user) { $this->user = $user; } @@ -119,6 +119,32 @@ abstract class AbstractImport implements ImportInterface $this->em->flush(); } + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) + { + foreach ($entries as $importedEntry) { + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->importedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } + /** * Parse one entry. * @@ -127,4 +153,14 @@ abstract class AbstractImport implements ImportInterface * @return Entry */ abstract public function parseEntry(array $importedEntry); + + /** + * Set current imported entry to archived / read. + * Implementation is different accross all imports. + * + * @param array $importedEntry + * + * @return array + */ + abstract protected function setEntryAsRead(array $importedEntry); } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 06a31813..5850deba 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -29,7 +29,7 @@ class PocketImport extends AbstractImport } /** - * Only used for test purpose + * Only used for test purpose. * * @return string */ @@ -258,24 +258,12 @@ class PocketImport extends AbstractImport } /** - * Faster parse entries for Producer. - * We don't care to make check at this time. They'll be done by the consumer. - * - * @param array $entries + * {@inheritdoc} */ - public function parseEntriesForProducer($entries) + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); - - if ($this->markAsRead) { - $importedEntry['status'] = 1; - } + $importedEntry['status'] = 1; - ++$this->importedEntries; - - $this->producer->publish(json_encode($importedEntry)); - } + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 18a6631a..64ef62bf 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -3,7 +3,6 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; class ReadabilityImport extends AbstractImport { @@ -136,31 +135,12 @@ class ReadabilityImport extends AbstractImport } /** - * Faster parse entries for Producer. - * We don't care to make check at this time. They'll be done by the consumer. - * - * @param array $entries + * {@inheritdoc} */ - protected function parseEntriesForProducer($entries) + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); - - if ($this->markAsRead) { - $importedEntry['archive'] = 1; - } - - ++$this->importedEntries; - - // flush every 20 entries - if (($i % 20) === 0) { - $this->em->flush(); - } - ++$i; - } + $importedEntry['archive'] = 1; - $this->em->flush(); - $this->em->clear(); + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 6ad14e8c..8e18e0ef 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -3,11 +3,9 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; abstract class WallabagImport extends AbstractImport { - protected $user; protected $skippedEntries = 0; protected $importedEntries = 0; protected $filepath; diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 86734652..292b72a7 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -57,19 +57,13 @@ class WallabagV1Import extends WallabagImport return $data; } - protected function parseEntriesForProducer($entries) + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); + $importedEntry['is_read'] = 1; - if ($this->markAsRead) { - $importedEntry['is_read'] = 1; - } - - ++$this->importedEntries; - - $this->producer->publish(json_encode($importedEntry)); - } + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php index faf4236f..37c8ca14 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php @@ -40,19 +40,13 @@ class WallabagV2Import extends WallabagImport ] + $entry; } - protected function parseEntriesForProducer($entries) + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) { - foreach ($entries as $importedEntry) { - // set userId for the producer (it won't know which user is connected) - $importedEntry['userId'] = $this->user->getId(); - - if ($this->markAsRead) { - $importedEntry['is_archived'] = 1; - } - - ++$this->importedEntries; + $importedEntry['is_archived'] = 1; - $this->producer->publish(json_encode($importedEntry)); - } + return $importedEntry; } } -- cgit v1.2.3 From 3aca0a9f00417b64203a660dee0a2b4c0fe22ac8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 5 Sep 2016 09:35:42 +0200 Subject: CS --- src/Wallabag/ImportBundle/Import/AbstractImport.php | 2 ++ src/Wallabag/ImportBundle/Import/PocketImport.php | 2 -- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 2 -- src/Wallabag/ImportBundle/Import/WallabagImport.php | 2 -- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index b085dc3a..8610062d 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -18,6 +18,8 @@ abstract class AbstractImport implements ImportInterface protected $producer; protected $user; protected $markAsRead; + protected $skippedEntries = 0; + protected $importedEntries = 0; public function __construct(EntityManager $em, ContentProxy $contentProxy) { diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 5850deba..845380b7 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -14,8 +14,6 @@ class PocketImport extends AbstractImport { private $client; private $consumerKey; - private $skippedEntries = 0; - private $importedEntries = 0; private $accessToken; const NB_ELEMENTS = 5000; diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 64ef62bf..915d4cd3 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -6,8 +6,6 @@ use Wallabag\CoreBundle\Entity\Entry; class ReadabilityImport extends AbstractImport { - private $skippedEntries = 0; - private $importedEntries = 0; private $filepath; /** diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 8e18e0ef..026567b0 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -6,8 +6,6 @@ use Wallabag\CoreBundle\Entity\Entry; abstract class WallabagImport extends AbstractImport { - protected $skippedEntries = 0; - protected $importedEntries = 0; protected $filepath; // untitled in all languages from v1 protected $untitled = [ -- cgit v1.2.3 From 6d65c0a8b089d3caa6f8e20d7935a9fe2f87d926 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 09:36:07 +0200 Subject: Add ability to define created_at for all import At the moment only Readability & wallabag v2 import allow created_at import. Pocket removed `time_added` field from their API v2 to v3... And wallabag v1 doesn't export that value. --- src/Wallabag/CoreBundle/Entity/Entry.php | 19 +++++++++++++++++-- src/Wallabag/ImportBundle/Import/PocketImport.php | 5 +++++ .../ImportBundle/Import/ReadabilityImport.php | 5 +++++ src/Wallabag/ImportBundle/Import/WallabagImport.php | 4 ++++ src/Wallabag/ImportBundle/Import/WallabagV1Import.php | 1 + .../Controller/ReadabilityControllerTest.php | 7 +++++++ .../Controller/WallabagV1ControllerTest.php | 6 ++++++ .../Controller/WallabagV2ControllerTest.php | 2 ++ 8 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/Wallabag/CoreBundle/Entity/Entry.php b/src/Wallabag/CoreBundle/Entity/Entry.php index c3e6b4d5..304258a9 100644 --- a/src/Wallabag/CoreBundle/Entity/Entry.php +++ b/src/Wallabag/CoreBundle/Entity/Entry.php @@ -410,7 +410,22 @@ class Entry } /** - * @return string + * Set created_at. + * Only used when importing data from an other service. + * + * @param DateTime $createdAt + * + * @return Entry + */ + public function setCreatedAt(\DateTime $createdAt) + { + $this->createdAt = $createdAt; + + return $this; + } + + /** + * @return DateTime */ public function getCreatedAt() { @@ -418,7 +433,7 @@ class Entry } /** - * @return string + * @return DateTime */ public function getUpdatedAt() { diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 845380b7..92dcdd40 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -193,6 +193,11 @@ class PocketImport extends AbstractImport $this->client = $client; } + /** + * {@inheritdoc} + * + * @see https://getpocket.com/developer/docs/v3/retrieve + */ public function parseEntry(array $importedEntry) { $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 915d4cd3..8f080d38 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -89,6 +89,9 @@ class ReadabilityImport extends AbstractImport return true; } + /** + * {@inheritdoc} + */ public function parseEntry(array $importedEntry) { $existingEntry = $this->em @@ -108,6 +111,7 @@ class ReadabilityImport extends AbstractImport 'language' => '', 'is_archived' => $importedEntry['archive'] || $this->markAsRead, 'is_starred' => $importedEntry['favorite'], + 'created_at' => $importedEntry['date_added'], ]; $entry = $this->fetchContent( @@ -125,6 +129,7 @@ class ReadabilityImport extends AbstractImport $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); + $entry->setCreatedAt(new \DateTime($data['created_at'])); $this->em->persist($entry); ++$this->importedEntries; diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 026567b0..8e50b135 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -139,6 +139,10 @@ abstract class WallabagImport extends AbstractImport $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); + if (!empty($data['created_at'])) { + $entry->setCreatedAt(new \DateTime($data['created_at'])); + } + $this->em->persist($entry); ++$this->importedEntries; diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 292b72a7..4f001062 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php @@ -42,6 +42,7 @@ class WallabagV1Import extends WallabagImport 'is_archived' => $entry['is_read'] || $this->markAsRead, 'is_starred' => $entry['is_fav'], 'tags' => '', + 'created_at' => '', ]; // force content to be refreshed in case on bad fetch in the v1 installation diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index 92cf4bfc..fb39356a 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -49,6 +49,13 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertNotEmpty($content->getLanguage()); + $this->assertEquals(0, count($content->getTags())); + $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt()); + $this->assertEquals('2016-08-25', $content->getCreatedAt()->format('Y-m-d')); } public function testImportReadabilityWithFileAndMarkAllAsRead() diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php index c1025b41..ff1bf6f0 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php @@ -56,6 +56,12 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertEmpty($content->getMimetype()); + $this->assertEmpty($content->getPreviewPicture()); + $this->assertEmpty($content->getLanguage()); + $this->assertEquals(1, count($content->getTags())); + $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt()); } public function testImportWallabagWithFileAndMarkAllAsRead() diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php index d8d2c8bf..149e88bb 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php @@ -67,6 +67,8 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $this->assertNotEmpty($content->getPreviewPicture()); $this->assertNotEmpty($content->getLanguage()); $this->assertEquals(2, count($content->getTags())); + $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt()); + $this->assertEquals('2016-09-08', $content->getCreatedAt()->format('Y-m-d')); } public function testImportWallabagWithEmptyFile() -- cgit v1.2.3 From 8664069e1aa2fa89e17587308a03f2720c20327a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 10:12:25 +0200 Subject: Fix DateTime & clear() --- src/Wallabag/CoreBundle/Entity/Entry.php | 10 +++++----- src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php | 7 ++++++- src/Wallabag/ImportBundle/Import/AbstractImport.php | 6 +++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/Wallabag/CoreBundle/Entity/Entry.php b/src/Wallabag/CoreBundle/Entity/Entry.php index 304258a9..a4b0d7a8 100644 --- a/src/Wallabag/CoreBundle/Entity/Entry.php +++ b/src/Wallabag/CoreBundle/Entity/Entry.php @@ -97,7 +97,7 @@ class Entry private $content; /** - * @var date + * @var \DateTime * * @ORM\Column(name="created_at", type="datetime") * @@ -106,7 +106,7 @@ class Entry private $createdAt; /** - * @var date + * @var \DateTime * * @ORM\Column(name="updated_at", type="datetime") * @@ -413,7 +413,7 @@ class Entry * Set created_at. * Only used when importing data from an other service. * - * @param DateTime $createdAt + * @param \DateTime $createdAt * * @return Entry */ @@ -425,7 +425,7 @@ class Entry } /** - * @return DateTime + * @return \DateTime */ public function getCreatedAt() { @@ -433,7 +433,7 @@ class Entry } /** - * @return DateTime + * @return \DateTime */ public function getUpdatedAt() { diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php index 8a8cf45d..72a3260a 100644 --- a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php +++ b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php @@ -7,6 +7,8 @@ use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface; use PhpAmqpLib\Message\AMQPMessage; use Wallabag\ImportBundle\Import\AbstractImport; use Wallabag\UserBundle\Repository\UserRepository; +use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\CoreBundle\Entity\Tag; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; @@ -53,7 +55,10 @@ class EntryConsumer implements ConsumerInterface try { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } catch (\Exception $e) { $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 8610062d..39befa7b 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -7,6 +7,7 @@ use Psr\Log\NullLogger; use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\Producer; @@ -113,7 +114,10 @@ abstract class AbstractImport implements ImportInterface // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - $this->em->clear($entry); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); } ++$i; } -- cgit v1.2.3 From 13470c3596d0b1490bbf18b39128a05bbb3c7f3e Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 18:02:29 +0200 Subject: Add test for RabbitMQ Also update Symfony deps --- app/config/parameters_test.yml | 2 +- src/Wallabag/ImportBundle/Import/PocketImport.php | 2 +- .../Consumer/AMPQ/EntryConsumerTest.php | 225 +++++++++++++++++++++ .../Controller/PocketControllerTest.php | 15 ++ .../Controller/ReadabilityControllerTest.php | 16 ++ .../Controller/WallabagV1ControllerTest.php | 16 ++ .../Controller/WallabagV2ControllerTest.php | 16 ++ .../ImportBundle/Import/PocketImportTest.php | 81 ++++++++ .../ImportBundle/Import/ReadabilityImportTest.php | 40 ++++ .../ImportBundle/Import/WallabagV1ImportTest.php | 40 ++++ .../ImportBundle/Import/WallabagV2ImportTest.php | 36 ++++ 11 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php diff --git a/app/config/parameters_test.yml b/app/config/parameters_test.yml index 696c95ef..2943b27a 100644 --- a/app/config/parameters_test.yml +++ b/app/config/parameters_test.yml @@ -5,4 +5,4 @@ parameters: test_database_name: null test_database_user: null test_database_password: null - test_database_path: '%kernel.root_dir%/../data/db/wallabag_testYO.sqlite' + test_database_path: '%kernel.root_dir%/../data/db/wallabag_test.sqlite' diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 92dcdd40..d76a3a08 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -265,7 +265,7 @@ class PocketImport extends AbstractImport */ protected function setEntryAsRead(array $importedEntry) { - $importedEntry['status'] = 1; + $importedEntry['status'] = '1'; return $importedEntry; } diff --git a/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php new file mode 100644 index 00000000..7141874c --- /dev/null +++ b/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php @@ -0,0 +1,225 @@ +getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->once()) + ->method('flush'); + + $em + ->expects($this->exactly(2)) + ->method('clear'); + + $body = <<<'JSON' +{ + "item_id": "1402935436", + "resolved_id": "1402935436", + "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "favorite": "0", + "status": "0", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, + "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...", + "is_article": "1", + "is_index": "0", + "has_video": "0", + "has_image": "1", + "word_count": "200", + "tags": { + "ifttt": { + "item_id": "1402935436", + "tag": "ifttt" + }, + "mashable": { + "item_id": "1402935436", + "tag": "mashable" + } + }, + "authors": { + "2484273": { + "item_id": "1402935436", + "author_id": "2484273", + "name": "Adam Rosenberg", + "url": "http://mashable.com/author/adam-rosenberg/" + } + }, + "image": { + "item_id": "1402935436", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0" + }, + "images": { + "1": { + "item_id": "1402935436", + "image_id": "1", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0", + "credit": "Image: Steve Eichner/NameFace/Sipa USA", + "caption": "" + } + }, + "userId": 1 +} +JSON; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(1) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn($entry); + + $consumer = new EntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } + + public function testMessageWithBadUser() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn(null); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $consumer = new EntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } + + public function testMessageWithEntryProcessed() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn(null); + + $consumer = new EntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php index e0e61df8..098cf356 100644 --- a/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php @@ -17,6 +17,21 @@ class PocketControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('button[type=submit]')->count()); } + public function testImportPocketWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + + $crawler = $client->request('GET', '/import/pocket'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('button[type=submit]')->count()); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + } + public function testImportPocketAuthBadToken() { $this->logInAs('admin'); diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index fb39356a..e12a723d 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -19,6 +19,22 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); } + public function testImportReadabilityWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + + $crawler = $client->request('GET', '/import/readability'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + } + public function testImportReadabilityWithFile() { $this->logInAs('admin'); diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php index ff1bf6f0..96556717 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php @@ -19,6 +19,22 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); } + public function testImportWallabagWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + + $crawler = $client->request('GET', '/import/wallabag-v1'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + } + public function testImportWallabagWithFile() { $this->logInAs('admin'); diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php index 149e88bb..250d0d3e 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php @@ -19,6 +19,22 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); } + public function testImportWallabagWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + + $crawler = $client->request('GET', '/import/wallabag-v2'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + } + public function testImportWallabagWithFile() { $this->logInAs('admin'); diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 26c6b825..5ad3e435 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -343,6 +343,87 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 2], $pocketImport->getSummary()); } + /** + * Will sample results from https://getpocket.com/developer/docs/v3/retrieve. + */ + public function testImportWithRabbit() + { + $client = new Client(); + + $body = <<<'JSON' +{ + "item_id": "229279689", + "resolved_id": "229279689", + "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", + "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", + "favorite": "1", + "status": "1", + "resolved_title": "The Massive Ryder Cup Preview", + "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", + "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", + "is_article": "1", + "has_video": "0", + "has_image": "0", + "word_count": "3197" +} +JSON; + + $mock = new Mock([ + new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))), + new Response(200, ['Content-Type' => 'application/json'], Stream::factory(' + { + "status": 1, + "list": { + "229279690": '.$body.' + } + } + ')), + ]); + + $client->getEmitter()->attach($mock); + + $pocketImport = $this->getPocketImport(); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = new Entry($this->user); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $bodyAsArray = json_decode($body, true); + // because with just use `new User()` so it doesn't have an id + $bodyAsArray['userId'] = null; + + $producer + ->expects($this->once()) + ->method('publish') + ->with(json_encode($bodyAsArray)); + + $pocketImport->setClient($client); + $pocketImport->setRabbitmqProducer($producer); + $pocketImport->authorize('wunderbar_code'); + + $res = $pocketImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary()); + } + public function testImportBadResponse() { $client = new Client(); diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php index 706d707b..69a66d6a 100644 --- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php @@ -120,6 +120,46 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); } + public function testImportWithRabbit() + { + $readabilityImport = $this->getReadabilityImport(); + $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(2)) + ->method('publish'); + + $readabilityImport->setRabbitmqProducer($producer); + + $res = $readabilityImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); + } + public function testImportBadFile() { $readabilityImport = $this->getReadabilityImport(); diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php index bdc47dac..ada5493e 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php @@ -120,6 +120,46 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 3], $wallabagV1Import->getSummary()); } + public function testImportWithRabbit() + { + $wallabagV1Import = $this->getWallabagV1Import(); + $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(4)) + ->method('publish'); + + $wallabagV1Import->setRabbitmqProducer($producer); + + $res = $wallabagV1Import->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary()); + } + public function testImportBadFile() { $wallabagV1Import = $this->getWallabagV1Import(); diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php index 4a45e0f0..51f0aada 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php @@ -116,6 +116,42 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 2], $wallabagV2Import->getSummary()); } + public function testImportWithRabbit() + { + $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(24)) + ->method('publish'); + + $wallabagV2Import->setRabbitmqProducer($producer); + + $res = $wallabagV2Import->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary()); + } + public function testImportBadFile() { $wallabagV1Import = $this->getWallabagV2Import(); -- cgit v1.2.3 From 7f7531171f6e49110b5842f869e37c766a682473 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 20:45:30 +0200 Subject: Retrieve created date from Pocket --- src/Wallabag/ImportBundle/Import/PocketImport.php | 4 ++++ .../ImportBundle/Import/PocketImportTest.php | 27 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index d76a3a08..fe39d33f 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -254,6 +254,10 @@ class PocketImport extends AbstractImport ); } + if (!empty($importedEntry['time_added'])) { + $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added'])); + } + $this->em->persist($entry); ++$this->importedEntries; diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 5ad3e435..1750e3a1 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -165,10 +165,16 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", "favorite": "1", "status": "1", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, "resolved_title": "The Massive Ryder Cup Preview", "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", "is_article": "1", + "is_index": "0", "has_video": "1", "has_image": "1", "word_count": "3197", @@ -212,10 +218,16 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", "favorite": "1", "status": "1", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 1, "resolved_title": "The Massive Ryder Cup Preview", "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", "is_article": "1", + "is_index": "0", "has_video": "0", "has_image": "0", "word_count": "3197" @@ -278,6 +290,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", "favorite": "1", "status": "1", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", "is_article": "1", "has_video": "1", @@ -291,6 +308,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", "favorite": "1", "status": "0", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 1, "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", "is_article": "1", "has_video": "0", @@ -358,6 +380,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", "favorite": "1", "status": "1", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, "resolved_title": "The Massive Ryder Cup Preview", "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", -- cgit v1.2.3 From b3437d58ae224121375c99e9288d8b808524e624 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 9 Sep 2016 21:02:03 +0200 Subject: Enable Redis async import - using javibravo/simpleue - internal config value are now `import_with_redis` & `import_with_rabbit` which are more clear - if both option are enable rabbit will be choosen - services imports related to async are now splitted into 2 files: `redis.yml` & `rabbit.yml` - --- app/config/config.yml | 8 +- app/config/parameters.yml.dist | 4 + composer.json | 7 +- src/Wallabag/CoreBundle/Command/InstallCommand.php | 7 +- .../DataFixtures/ORM/LoadSettingData.php | 7 +- .../CoreBundle/Resources/config/services.yml | 8 + .../ImportBundle/Command/RedisWorkerCommand.php | 41 ++++ .../ImportBundle/Consumer/AMPQ/EntryConsumer.php | 68 ------- .../ImportBundle/Consumer/AMPQEntryConsumer.php | 70 +++++++ .../ImportBundle/Consumer/RedisEntryConsumer.php | 84 ++++++++ .../ImportBundle/Controller/PocketController.php | 6 +- .../Controller/ReadabilityController.php | 6 +- .../Controller/WallabagV1Controller.php | 6 +- .../Controller/WallabagV2Controller.php | 6 +- .../ImportBundle/Import/AbstractImport.php | 8 +- src/Wallabag/ImportBundle/Redis/Producer.php | 36 ++++ .../ImportBundle/Resources/config/rabbit.yml | 30 +++ .../ImportBundle/Resources/config/redis.yml | 81 ++++++++ .../ImportBundle/Resources/config/services.yml | 33 +-- .../Consumer/AMPQ/EntryConsumerTest.php | 225 --------------------- .../Consumer/AMPQEntryConsumerTest.php | 225 +++++++++++++++++++++ .../Consumer/RedisEntryConsumerTest.php | 224 ++++++++++++++++++++ .../Controller/PocketControllerTest.php | 19 +- .../Controller/ReadabilityControllerTest.php | 20 +- .../Controller/WallabagV1ControllerTest.php | 20 +- .../Controller/WallabagV2ControllerTest.php | 20 +- .../ImportBundle/Import/PocketImportTest.php | 86 +++++++- .../ImportBundle/Import/ReadabilityImportTest.php | 45 ++++- .../ImportBundle/Import/WallabagV1ImportTest.php | 45 ++++- .../ImportBundle/Import/WallabagV2ImportTest.php | 41 +++- 30 files changed, 1132 insertions(+), 354 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php delete mode 100644 src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php create mode 100644 src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php create mode 100644 src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php create mode 100644 src/Wallabag/ImportBundle/Redis/Producer.php create mode 100644 src/Wallabag/ImportBundle/Resources/config/rabbit.yml create mode 100644 src/Wallabag/ImportBundle/Resources/config/redis.yml delete mode 100644 tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php create mode 100644 tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php create mode 100644 tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php diff --git a/app/config/config.yml b/app/config/config.yml index d39bef95..e18a932d 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -254,7 +254,7 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.pocket' - callback: wallabag_import.consumer.pocket + callback: wallabag_import.consumer.ampq.pocket import_readability: connection: default exchange_options: @@ -262,7 +262,7 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.readability' - callback: wallabag_import.consumer.readability + callback: wallabag_import.consumer.ampq.readability import_wallabag_v1: connection: default exchange_options: @@ -270,7 +270,7 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.wallabag_v1' - callback: wallabag_import.consumer.wallabag_v1 + callback: wallabag_import.consumer.ampq.wallabag_v1 import_wallabag_v2: connection: default exchange_options: @@ -278,4 +278,4 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.wallabag_v2' - callback: wallabag_import.consumer.wallabag_v2 + callback: wallabag_import.consumer.ampq.wallabag_v2 diff --git a/app/config/parameters.yml.dist b/app/config/parameters.yml.dist index a59dc02c..f2e5bec3 100644 --- a/app/config/parameters.yml.dist +++ b/app/config/parameters.yml.dist @@ -46,3 +46,7 @@ parameters: rabbitmq_port: 5672 rabbitmq_user: guest rabbitmq_password: guest + + # Redis processing + redis_host: localhost + redis_port: 6379 diff --git a/composer.json b/composer.json index 121a3fd3..8220ed45 100644 --- a/composer.json +++ b/composer.json @@ -82,7 +82,9 @@ "ocramius/proxy-manager": "1.*", "white-october/pagerfanta-bundle": "^1.0", "mouf/nodejs-installer": "~1.0", - "php-amqplib/rabbitmq-bundle": "^1.8" + "php-amqplib/rabbitmq-bundle": "^1.8", + "predis/predis": "^1.0", + "javibravo/simpleue": "^1.0" }, "require-dev": { "doctrine/doctrine-fixtures-bundle": "~2.2", @@ -90,7 +92,8 @@ "sensio/generator-bundle": "^3.0", "phpunit/phpunit": "~5.0", "symfony/phpunit-bridge": "^3.0", - "friendsofphp/php-cs-fixer": "~1.9" + "friendsofphp/php-cs-fixer": "~1.9", + "m6web/redis-mock": "^2.0" }, "scripts": { "post-cmd": [ diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 8a2439ec..4b1ec02a 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -322,7 +322,12 @@ class InstallCommand extends ContainerAwareCommand 'section' => 'import', ], [ - 'name' => 'rabbitmq', + 'name' => 'import_with_redis', + 'value' => '0', + 'section' => 'import', + ], + [ + 'name' => 'import_with_rabbitmq', 'value' => '0', 'section' => 'import', ], diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index 778f91ed..3ed6bafe 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -96,7 +96,12 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface 'section' => 'import', ], [ - 'name' => 'rabbitmq', + 'name' => 'import_with_redis', + 'value' => '0', + 'section' => 'import', + ], + [ + 'name' => 'import_with_rabbitmq', 'value' => '0', 'section' => 'import', ], diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index e95ef452..23e6d3ca 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -125,3 +125,11 @@ services: arguments: - "@security.token_storage" - "@router" + + wallabag_core.redis.client: + class: Predis\Client + arguments: + - + host: '%redis_host%' + port: '%redis_port%' + schema: tcp diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php new file mode 100644 index 00000000..85c5a903 --- /dev/null +++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php @@ -0,0 +1,41 @@ +setName('wallabag:import:redis-worker') + ->setDescription('Launch Redis worker') + ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s')); + $output->writeln('Waiting for message ...'); + + $serviceName = $input->getArgument('serviceName'); + + if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) { + throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName'))); + } + + $worker = new QueueWorker( + $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName), + $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName) + ); + + $worker->start(); + } +} diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php deleted file mode 100644 index 72a3260a..00000000 --- a/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php +++ /dev/null @@ -1,68 +0,0 @@ -em = $em; - $this->userRepository = $userRepository; - $this->import = $import; - $this->logger = $logger ?: new NullLogger(); - } - - /** - * {@inheritdoc} - */ - public function execute(AMQPMessage $msg) - { - $storedEntry = json_decode($msg->body, true); - - $user = $this->userRepository->find($storedEntry['userId']); - - // no user? Drop message - if (null === $user) { - $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); - - return; - } - - $this->import->setUser($user); - - $entry = $this->import->parseEntry($storedEntry); - - if (null === $entry) { - $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); - - return; - } - - try { - $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); - } catch (\Exception $e) { - $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); - - return; - } - } -} diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php new file mode 100644 index 00000000..39bb5375 --- /dev/null +++ b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php @@ -0,0 +1,70 @@ +em = $em; + $this->userRepository = $userRepository; + $this->import = $import; + $this->logger = $logger ?: new NullLogger(); + } + + /** + * {@inheritdoc} + */ + public function execute(AMQPMessage $msg) + { + $storedEntry = json_decode($msg->body, true); + + $user = $this->userRepository->find($storedEntry['userId']); + + // no user? Drop message + if (null === $user) { + $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); + + return; + } + + $this->import->setUser($user); + + $entry = $this->import->parseEntry($storedEntry); + + if (null === $entry) { + $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); + + return; + } + + try { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } catch (\Exception $e) { + $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); + + return; + } + + $this->logger->info('Content with url ('.$entry->getUrl().') imported !'); + } +} diff --git a/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php new file mode 100644 index 00000000..38665b01 --- /dev/null +++ b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php @@ -0,0 +1,84 @@ +em = $em; + $this->userRepository = $userRepository; + $this->import = $import; + $this->logger = $logger ?: new NullLogger(); + } + + /** + * Handle one message by one message. + * + * @param string $job Content of the message (directly from Redis) + * + * @return bool + */ + public function manage($job) + { + $storedEntry = json_decode($job, true); + + $user = $this->userRepository->find($storedEntry['userId']); + + // no user? Drop message + if (null === $user) { + $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); + + return false; + } + + $this->import->setUser($user); + + $entry = $this->import->parseEntry($storedEntry); + + if (null === $entry) { + $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); + + return false; + } + + try { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } catch (\Exception $e) { + $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); + + return false; + } + + $this->logger->info('Content with url ('.$entry->getUrl().') imported !'); + + return true; + } + + /** + * Should tell if the given job will kill the worker. + * We don't want to stop it :). + */ + public function isStopJob($job) + { + return false; + } +} diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 57c007c3..3d555717 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -20,8 +20,10 @@ class PocketController extends Controller $pocket = $this->get('wallabag_import.pocket.import'); $pocket->setUser($this->getUser()); - if ($this->get('craue_config')->get('rabbitmq')) { - $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket')); } return $pocket; diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index ee875a40..61243042 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -20,8 +20,10 @@ class ReadabilityController extends Controller $readability = $this->get('wallabag_import.readability.import'); $readability->setUser($this->getUser()); - if ($this->get('craue_config')->get('rabbitmq')) { - $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $readability->setProducer($this->get('wallabag_import.producer.redis.readability')); } if ($form->isValid()) { diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php index f80aec3a..312c7a35 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php @@ -14,8 +14,10 @@ class WallabagV1Controller extends WallabagController { $service = $this->get('wallabag_import.wallabag_v1.import'); - if ($this->get('craue_config')->get('rabbitmq')) { - $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1')); } return $service; diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php index 063cddd9..45211fe6 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php @@ -14,8 +14,10 @@ class WallabagV2Controller extends WallabagController { $service = $this->get('wallabag_import.wallabag_v2.import'); - if ($this->get('craue_config')->get('rabbitmq')) { - $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2')); } return $service; diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 39befa7b..4cd8e846 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -9,7 +9,7 @@ use Wallabag\CoreBundle\Helper\ContentProxy; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; -use OldSound\RabbitMqBundle\RabbitMq\Producer; +use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; abstract class AbstractImport implements ImportInterface { @@ -35,12 +35,12 @@ abstract class AbstractImport implements ImportInterface } /** - * Set RabbitMQ Producer to send each entry to a queue. + * Set RabbitMQ/Redis Producer to send each entry to a queue. * This method should be called when user has enabled RabbitMQ. * - * @param Producer $producer + * @param ProducerInterface $producer */ - public function setRabbitmqProducer(Producer $producer) + public function setProducer(ProducerInterface $producer) { $this->producer = $producer; } diff --git a/src/Wallabag/ImportBundle/Redis/Producer.php b/src/Wallabag/ImportBundle/Redis/Producer.php new file mode 100644 index 00000000..fedc3e57 --- /dev/null +++ b/src/Wallabag/ImportBundle/Redis/Producer.php @@ -0,0 +1,36 @@ +queue = $queue; + } + + /** + * Publish a message in the Redis queue. + * + * @param string $msgBody + * @param string $routingKey NOT USED + * @param array $additionalProperties NOT USED + */ + public function publish($msgBody, $routingKey = '', $additionalProperties = array()) + { + $this->queue->sendJob($msgBody); + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml new file mode 100644 index 00000000..f09dda0d --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -0,0 +1,30 @@ +# RabbitMQ stuff +services: + wallabag_import.consumer.ampq.pocket: + class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.pocket.import" + - "@logger" + wallabag_import.consumer.ampq.readability: + class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.readability.import" + - "@logger" + wallabag_import.consumer.ampq.wallabag_v1: + class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.wallabag_v1.import" + - "@logger" + wallabag_import.consumer.ampq.wallabag_v2: + class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.wallabag_v2.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml new file mode 100644 index 00000000..7d3248e5 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -0,0 +1,81 @@ +# Redis stuff +services: + # readability + wallabag_import.queue.redis.readability: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.readability" + + wallabag_import.producer.redis.readability: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.readability" + + wallabag_import.consumer.redis.readability: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.readability.import" + - "@logger" + + # pocket + wallabag_import.queue.redis.pocket: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.pocket" + + wallabag_import.producer.redis.pocket: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.pocket" + + wallabag_import.consumer.redis.pocket: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.pocket.import" + - "@logger" + + # wallabag v1 + wallabag_import.queue.redis.wallabag_v1: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.wallabag_v1" + + wallabag_import.producer.redis.wallabag_v1: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.wallabag_v1" + + wallabag_import.consumer.redis.wallabag_v1: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.wallabag_v1.import" + - "@logger" + + # wallabag v2 + wallabag_import.queue.redis.wallabag_v2: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.wallabag_v2" + + wallabag_import.producer.redis.wallabag_v2: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.wallabag_v2" + + wallabag_import.consumer.redis.wallabag_v2: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.wallabag_v2.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index cad44e71..f03404ae 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -1,33 +1,8 @@ -services: - wallabag_import.consumer.pocket: - class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer - arguments: - - "@doctrine.orm.entity_manager" - - "@wallabag_user.user_repository" - - "@wallabag_import.pocket.import" - - "@logger" - wallabag_import.consumer.readability: - class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer - arguments: - - "@doctrine.orm.entity_manager" - - "@wallabag_user.user_repository" - - "@wallabag_import.readability.import" - - "@logger" - wallabag_import.consumer.wallabag_v1: - class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer - arguments: - - "@doctrine.orm.entity_manager" - - "@wallabag_user.user_repository" - - "@wallabag_import.wallabag_v1.import" - - "@logger" - wallabag_import.consumer.wallabag_v2: - class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer - arguments: - - "@doctrine.orm.entity_manager" - - "@wallabag_user.user_repository" - - "@wallabag_import.wallabag_v2.import" - - "@logger" +imports: + - { resource: rabbit.yml } + - { resource: redis.yml } +services: wallabag_import.chain: class: Wallabag\ImportBundle\Import\ImportChain diff --git a/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php deleted file mode 100644 index 7141874c..00000000 --- a/tests/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumerTest.php +++ /dev/null @@ -1,225 +0,0 @@ -getMockBuilder('Doctrine\ORM\EntityManager') - ->disableOriginalConstructor() - ->getMock(); - - $em - ->expects($this->once()) - ->method('flush'); - - $em - ->expects($this->exactly(2)) - ->method('clear'); - - $body = <<<'JSON' -{ - "item_id": "1402935436", - "resolved_id": "1402935436", - "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", - "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules", - "favorite": "0", - "status": "0", - "time_added": "1473020899", - "time_updated": "1473020899", - "time_read": "0", - "time_favorited": "0", - "sort_id": 0, - "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules", - "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", - "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...", - "is_article": "1", - "is_index": "0", - "has_video": "0", - "has_image": "1", - "word_count": "200", - "tags": { - "ifttt": { - "item_id": "1402935436", - "tag": "ifttt" - }, - "mashable": { - "item_id": "1402935436", - "tag": "mashable" - } - }, - "authors": { - "2484273": { - "item_id": "1402935436", - "author_id": "2484273", - "name": "Adam Rosenberg", - "url": "http://mashable.com/author/adam-rosenberg/" - } - }, - "image": { - "item_id": "1402935436", - "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", - "width": "0", - "height": "0" - }, - "images": { - "1": { - "item_id": "1402935436", - "image_id": "1", - "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", - "width": "0", - "height": "0", - "credit": "Image: Steve Eichner/NameFace/Sipa USA", - "caption": "" - } - }, - "userId": 1 -} -JSON; - - $user = new User(); - $entry = new Entry($user); - - $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') - ->disableOriginalConstructor() - ->getMock(); - - $userRepository - ->expects($this->once()) - ->method('find') - // userId from the body json above - ->with(1) - ->willReturn($user); - - $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') - ->disableOriginalConstructor() - ->getMock(); - - $import - ->expects($this->once()) - ->method('setUser') - ->with($user); - - $import - ->expects($this->once()) - ->method('parseEntry') - ->with(json_decode($body, true)) - ->willReturn($entry); - - $consumer = new EntryConsumer( - $em, - $userRepository, - $import - ); - - $message = new AMQPMessage($body); - - $consumer->execute($message); - } - - public function testMessageWithBadUser() - { - $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') - ->disableOriginalConstructor() - ->getMock(); - - $em - ->expects($this->never()) - ->method('flush'); - - $em - ->expects($this->never()) - ->method('clear'); - - $body = '{ "userId": 123 }'; - - $user = new User(); - $entry = new Entry($user); - - $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') - ->disableOriginalConstructor() - ->getMock(); - - $userRepository - ->expects($this->once()) - ->method('find') - // userId from the body json above - ->with(123) - ->willReturn(null); - - $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') - ->disableOriginalConstructor() - ->getMock(); - - $consumer = new EntryConsumer( - $em, - $userRepository, - $import - ); - - $message = new AMQPMessage($body); - - $consumer->execute($message); - } - - public function testMessageWithEntryProcessed() - { - $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') - ->disableOriginalConstructor() - ->getMock(); - - $em - ->expects($this->never()) - ->method('flush'); - - $em - ->expects($this->never()) - ->method('clear'); - - $body = '{ "userId": 123 }'; - - $user = new User(); - - $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') - ->disableOriginalConstructor() - ->getMock(); - - $userRepository - ->expects($this->once()) - ->method('find') - // userId from the body json above - ->with(123) - ->willReturn($user); - - $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') - ->disableOriginalConstructor() - ->getMock(); - - $import - ->expects($this->once()) - ->method('setUser') - ->with($user); - - $import - ->expects($this->once()) - ->method('parseEntry') - ->with(json_decode($body, true)) - ->willReturn(null); - - $consumer = new EntryConsumer( - $em, - $userRepository, - $import - ); - - $message = new AMQPMessage($body); - - $consumer->execute($message); - } -} diff --git a/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php new file mode 100644 index 00000000..b13ade1d --- /dev/null +++ b/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php @@ -0,0 +1,225 @@ +getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->once()) + ->method('flush'); + + $em + ->expects($this->exactly(2)) + ->method('clear'); + + $body = <<<'JSON' +{ + "item_id": "1402935436", + "resolved_id": "1402935436", + "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "favorite": "0", + "status": "0", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, + "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...", + "is_article": "1", + "is_index": "0", + "has_video": "0", + "has_image": "1", + "word_count": "200", + "tags": { + "ifttt": { + "item_id": "1402935436", + "tag": "ifttt" + }, + "mashable": { + "item_id": "1402935436", + "tag": "mashable" + } + }, + "authors": { + "2484273": { + "item_id": "1402935436", + "author_id": "2484273", + "name": "Adam Rosenberg", + "url": "http://mashable.com/author/adam-rosenberg/" + } + }, + "image": { + "item_id": "1402935436", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0" + }, + "images": { + "1": { + "item_id": "1402935436", + "image_id": "1", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0", + "credit": "Image: Steve Eichner/NameFace/Sipa USA", + "caption": "" + } + }, + "userId": 1 +} +JSON; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(1) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn($entry); + + $consumer = new AMPQEntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } + + public function testMessageWithBadUser() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn(null); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $consumer = new AMPQEntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } + + public function testMessageWithEntryProcessed() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn(null); + + $consumer = new AMPQEntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } +} diff --git a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php new file mode 100644 index 00000000..0ce7ce49 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php @@ -0,0 +1,224 @@ +getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->once()) + ->method('flush'); + + $em + ->expects($this->exactly(2)) + ->method('clear'); + + $body = <<<'JSON' +{ + "item_id": "1402935436", + "resolved_id": "1402935436", + "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "favorite": "0", + "status": "0", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, + "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...", + "is_article": "1", + "is_index": "0", + "has_video": "0", + "has_image": "1", + "word_count": "200", + "tags": { + "ifttt": { + "item_id": "1402935436", + "tag": "ifttt" + }, + "mashable": { + "item_id": "1402935436", + "tag": "mashable" + } + }, + "authors": { + "2484273": { + "item_id": "1402935436", + "author_id": "2484273", + "name": "Adam Rosenberg", + "url": "http://mashable.com/author/adam-rosenberg/" + } + }, + "image": { + "item_id": "1402935436", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0" + }, + "images": { + "1": { + "item_id": "1402935436", + "image_id": "1", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0", + "credit": "Image: Steve Eichner/NameFace/Sipa USA", + "caption": "" + } + }, + "userId": 1 +} +JSON; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(1) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn($entry); + + $consumer = new RedisEntryConsumer( + $em, + $userRepository, + $import + ); + + $res = $consumer->manage($body); + + $this->assertTrue($res); + } + + public function testMessageWithBadUser() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn(null); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $consumer = new RedisEntryConsumer( + $em, + $userRepository, + $import + ); + + $res = $consumer->manage($body); + + $this->assertFalse($res); + } + + public function testMessageWithEntryProcessed() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn(null); + + $consumer = new RedisEntryConsumer( + $em, + $userRepository, + $import + ); + + $res = $consumer->manage($body); + + $this->assertFalse($res); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php index 098cf356..35673261 100644 --- a/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/PocketControllerTest.php @@ -22,14 +22,29 @@ class PocketControllerTest extends WallabagCoreTestCase $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); $crawler = $client->request('GET', '/import/pocket'); $this->assertEquals(200, $client->getResponse()->getStatusCode()); $this->assertEquals(1, $crawler->filter('button[type=submit]')->count()); - $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportPocketWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/pocket'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('button[type=submit]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportPocketAuthBadToken() diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index e12a723d..69635382 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -24,7 +24,7 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); $crawler = $client->request('GET', '/import/readability'); @@ -32,7 +32,23 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); - $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportReadabilityWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/readability'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportReadabilityWithFile() diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php index 96556717..933ddd6c 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php @@ -24,7 +24,7 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); $crawler = $client->request('GET', '/import/wallabag-v1'); @@ -32,7 +32,23 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); - $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportWallabagWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/wallabag-v1'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportWallabagWithFile() diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php index 250d0d3e..36e5221d 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php @@ -24,7 +24,7 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('rabbitmq', 1); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); $crawler = $client->request('GET', '/import/wallabag-v2'); @@ -32,7 +32,23 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); - $client->getContainer()->get('craue_config')->set('rabbitmq', 0); + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportWallabagWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/wallabag-v2'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportWallabagWithFile() diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 1750e3a1..425fa321 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -9,8 +9,11 @@ use GuzzleHttp\Client; use GuzzleHttp\Subscriber\Mock; use GuzzleHttp\Message\Response; use GuzzleHttp\Stream\Stream; +use Wallabag\ImportBundle\Redis\Producer; use Monolog\Logger; use Monolog\Handler\TestHandler; +use Simpleue\Queue\RedisQueue; +use M6Web\Component\RedisMock\RedisMockFactory; class PocketImportTest extends \PHPUnit_Framework_TestCase { @@ -442,7 +445,7 @@ JSON; ->with(json_encode($bodyAsArray)); $pocketImport->setClient($client); - $pocketImport->setRabbitmqProducer($producer); + $pocketImport->setProducer($producer); $pocketImport->authorize('wunderbar_code'); $res = $pocketImport->setMarkAsRead(true)->import(); @@ -451,6 +454,87 @@ JSON; $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary()); } + /** + * Will sample results from https://getpocket.com/developer/docs/v3/retrieve. + */ + public function testImportWithRedis() + { + $client = new Client(); + + $body = <<<'JSON' +{ + "item_id": "229279689", + "resolved_id": "229279689", + "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", + "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", + "favorite": "1", + "status": "1", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, + "resolved_title": "The Massive Ryder Cup Preview", + "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", + "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", + "is_article": "1", + "has_video": "0", + "has_image": "0", + "word_count": "3197" +} +JSON; + + $mock = new Mock([ + new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))), + new Response(200, ['Content-Type' => 'application/json'], Stream::factory(' + { + "status": 1, + "list": { + "229279690": '.$body.' + } + } + ')), + ]); + + $client->getEmitter()->attach($mock); + + $pocketImport = $this->getPocketImport(); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = new Entry($this->user); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'pocket'); + $producer = new Producer($queue); + + $pocketImport->setClient($client); + $pocketImport->setProducer($producer); + $pocketImport->authorize('wunderbar_code'); + + $res = $pocketImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('pocket')); + } + public function testImportBadResponse() { $client = new Client(); diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php index 69a66d6a..0981eedb 100644 --- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php @@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import; use Wallabag\ImportBundle\Import\ReadabilityImport; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\ImportBundle\Redis\Producer; use Monolog\Logger; use Monolog\Handler\TestHandler; +use Simpleue\Queue\RedisQueue; +use M6Web\Component\RedisMock\RedisMockFactory; class ReadabilityImportTest extends \PHPUnit_Framework_TestCase { @@ -152,7 +155,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->expects($this->exactly(2)) ->method('publish'); - $readabilityImport->setRabbitmqProducer($producer); + $readabilityImport->setProducer($producer); $res = $readabilityImport->setMarkAsRead(true)->import(); @@ -160,6 +163,46 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); } + public function testImportWithRedis() + { + $readabilityImport = $this->getReadabilityImport(); + $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'readability'); + $producer = new Producer($queue); + + $readabilityImport->setProducer($producer); + + $res = $readabilityImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('readability')); + } + public function testImportBadFile() { $readabilityImport = $this->getReadabilityImport(); diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php index ada5493e..b43682cd 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php @@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import; use Wallabag\ImportBundle\Import\WallabagV1Import; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\ImportBundle\Redis\Producer; use Monolog\Logger; use Monolog\Handler\TestHandler; +use Simpleue\Queue\RedisQueue; +use M6Web\Component\RedisMock\RedisMockFactory; class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase { @@ -152,7 +155,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase ->expects($this->exactly(4)) ->method('publish'); - $wallabagV1Import->setRabbitmqProducer($producer); + $wallabagV1Import->setProducer($producer); $res = $wallabagV1Import->setMarkAsRead(true)->import(); @@ -160,6 +163,46 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary()); } + public function testImportWithRedis() + { + $wallabagV1Import = $this->getWallabagV1Import(); + $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'wallabag_v1'); + $producer = new Producer($queue); + + $wallabagV1Import->setProducer($producer); + + $res = $wallabagV1Import->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('wallabag_v1')); + } + public function testImportBadFile() { $wallabagV1Import = $this->getWallabagV1Import(); diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php index 51f0aada..18998b35 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php @@ -5,8 +5,11 @@ namespace Tests\Wallabag\ImportBundle\Import; use Wallabag\ImportBundle\Import\WallabagV2Import; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\ImportBundle\Redis\Producer; use Monolog\Logger; use Monolog\Handler\TestHandler; +use Simpleue\Queue\RedisQueue; +use M6Web\Component\RedisMock\RedisMockFactory; class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase { @@ -144,7 +147,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase ->expects($this->exactly(24)) ->method('publish'); - $wallabagV2Import->setRabbitmqProducer($producer); + $wallabagV2Import->setProducer($producer); $res = $wallabagV2Import->setMarkAsRead(true)->import(); @@ -152,6 +155,42 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary()); } + public function testImportWithRedis() + { + $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'wallabag_v2'); + $producer = new Producer($queue); + + $wallabagV2Import->setProducer($producer); + + $res = $wallabagV2Import->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('wallabag_v2')); + } + public function testImportBadFile() { $wallabagV1Import = $this->getWallabagV2Import(); -- cgit v1.2.3 From 7230e4c39f84e5aca97f439953adfd265b8d9ba4 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 18:19:41 +0200 Subject: Enable Redis on Travis Add generated files from `composer up` Add more articles for Readability tests --- .travis.yml | 1 + .../ImportBundle/Import/ReadabilityImportTest.php | 20 +-- .../ImportBundle/fixtures/readability.json | 167 ++++++++++++++++++++- 3 files changed, 170 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 918d15fe..c5fe7b01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: php services: - rabbitmq + - redis # faster builds on docker-container setup sudo: false diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php index 0981eedb..67917154 100644 --- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php @@ -61,9 +61,9 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $entryRepo->expects($this->exactly(2)) + $entryRepo->expects($this->exactly(24)) ->method('findByUrlAndUserId') - ->will($this->onConsecutiveCalls(false, true)); + ->willReturn(false); $this->em ->expects($this->any()) @@ -75,14 +75,14 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->getMock(); $this->contentProxy - ->expects($this->exactly(1)) + ->expects($this->exactly(24)) ->method('updateEntry') ->willReturn($entry); $res = $readabilityImport->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 1, 'imported' => 1], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 24], $readabilityImport->getSummary()); } public function testImportAndMarkAllAsRead() @@ -96,7 +96,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $entryRepo->expects($this->exactly(2)) ->method('findByUrlAndUserId') - ->will($this->onConsecutiveCalls(false, false)); + ->will($this->onConsecutiveCalls(false, true)); $this->em ->expects($this->any()) @@ -104,7 +104,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->willReturn($entryRepo); $this->contentProxy - ->expects($this->exactly(2)) + ->expects($this->exactly(1)) ->method('updateEntry') ->willReturn(new Entry($this->user)); @@ -120,7 +120,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 1, 'imported' => 1], $readabilityImport->getSummary()); } public function testImportWithRabbit() @@ -152,7 +152,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->getMock(); $producer - ->expects($this->exactly(2)) + ->expects($this->exactly(24)) ->method('publish'); $readabilityImport->setProducer($producer); @@ -160,7 +160,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $res = $readabilityImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 24], $readabilityImport->getSummary()); } public function testImportWithRedis() @@ -198,7 +198,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $res = $readabilityImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 2], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 24], $readabilityImport->getSummary()); $this->assertNotEmpty($redisMock->lpop('readability')); } diff --git a/tests/Wallabag/ImportBundle/fixtures/readability.json b/tests/Wallabag/ImportBundle/fixtures/readability.json index 34379905..32f6fa53 100644 --- a/tests/Wallabag/ImportBundle/fixtures/readability.json +++ b/tests/Wallabag/ImportBundle/fixtures/readability.json @@ -11,14 +11,165 @@ "archive": false }, { - "article__excerpt": "TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data. Here’s a demo and the code for the infinite…", - "favorite": false, - "date_archived": "2016-08-26T12:21:54", - "article__url": "https://developers.google.com/web/updates/2016/07/infinite-scroller?imm_mid=0e6839&cmp=em-webops-na-na-newsltr_20160805", - "date_added": "2016-08-06T05:35:26", - "date_favorited": null, - "article__title": "Complexities of an infinite scroller | Web Updates", - "archive": true + "article__title": "Réfugiés: l'UE va créer 100 000 places d'accueil dans les Balkans", + "article__url": "http://www.liberation.fr/planete/2015/10/26/refugies-l-ue-va-creer-100-000-places-d-accueil-dans-les-balkans_1408867", + "archive": false, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": false + }, + { + "article__title": "No title found", + "article__url": "http://news.nationalgeographic.com/2016/02/160211-albatrosses-mothers-babies-animals-science/&sf20739758=1", + "archive": false, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": true + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Échecs", + "article__url": "https://fr.wikipedia.org/wiki/Échecs" + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "90% des dossiers médicaux des Coréens du sud vendus à des entreprises privées - ZATAZ", + "article__url": "http://www.zataz.com/90-des-dossiers-medicaux-des-coreens-du-sud-vendus-a-des-entreprises-privees/" + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Mass Surveillance As Art", + "article__url": "https://www.nationaljournal.com/s/73311/mass-surveillance-art" + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "What David Cameron did to the pig, his party is now doing to the country", + "article__url": "http://www.newstatesman.com/2015/09/what-david-cameron-did-pig-his-party-now-doing-country" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "CLICK HERE to support 2016 CES Winner, Revolutionary Auto-Tracking Robot", + "article__url": "https://www.indiegogo.com/projects/2016-ces-winner-revolutionary-auto-tracking-robot" + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 1, + "article__title": "No title found", + "article__url": "http://carnetdevol.shost.ca/wordpress/aide-memoire-sur-les-commandes-associees-a-systemd/" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Présentation d'Arduino - Tuto Arduino - Le blog d'Eskimon", + "article__url": "http://eskimon.fr/73-arduino-101-presentation" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Lenovo ThinkPad X1 Carbon Ultrabook Review", + "article__url": "http://www.notebookcheck.net/Lenovo-ThinkPad-X1-Carbon-Ultrabook-Review.138033.0.html" + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Visitons le Château de Landsberg !", + "article__url": "http://autour-du-mont-sainte-odile.overblog.com/2016/01/visitons-le-chateau-de-landsberg.html" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Contrer les stéréotypes par les livres : “C'est dès l'enfance qu'ils se construisent”", + "article__url": "https://www.actualitte.com/article/monde-edition/contrer-les-stereotypes-par-les-livres-c-est-des-l-enfance-qu-ils-se-construisent/64058" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "[ROM][6.0.1][Layers][N5] TipsyOS official builds {UBER TCs}", + "article__url": "http://forum.xda-developers.com/google-nexus-5/development/rom-tipsyos-official-builds-uber-tcs-t3325989" + }, + { + "archive": 0, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Top 15 Podcasts All Web Developers Should Follow - Envato Tuts+ Code Article", + "article__url": "http://code.tutsplus.com/articles/top-15-podcasts-all-web-developers-should-follow--net-14461" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "University of Mississippi", + "article__url": "http://olemiss.edu" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "FinnChristiansen.de Jetzt Dank Let’s Encrypt Per HTTPS Erreichbar", + "article__url": "https://www.finnchristiansen.de/2015/12/06/finnchristiansen-de-jetzt-dank-lets-encrypt-per-https-erreichbar/" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Le développeur et l'ingénierie logicielle", + "article__url": "http://wemucs.com/le-developpeur-et-lingenierie-logicielle/" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "The Role of Methylation in Gene Expression", + "article__url": "http://www.nature.com/scitable/topicpage/the-role-of-methylation-in-gene-expression-1070" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "E-Mail-Adresse kostenlos, FreeMail, De-Mail & Nachrichten", + "article__url": "http://web.de" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "OpenSSH Server on Arch Linux | DominicM test", + "article__url": "http://dominicm.com/openssh-server-arch-linux/" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Site Moved | Site Help", + "article__url": "http://g1.com/help/sitemoved.asp" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "#Maroc : le stylo anti-pédophiles EAGLE d’AMESYS est moins bien configuré que les faux-lowers Twitter du roi Mohammed VI", + "article__url": "https://reflets.info/maroc-le-stylo-anti-pedophiles-eagle-damesys-est-moins-bien-configure-que-les-faux-lowers-twitter-du-roi-mohammed-vi/" + }, + { + "archive": 1, + "date_added": "2016-09-08T11:55:58+0200", + "favorite": 0, + "article__title": "Simple Cloud Infrastructure for Developers", + "article__url": "https://www.digitalocean.com/" } ], "recommendations": [] -- cgit v1.2.3 From dc69e25f97c357fdfdff5225f4f65cc55a6770b0 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 20:22:38 +0200 Subject: Display a message when async is enabled --- src/Wallabag/CoreBundle/Resources/translations/messages.da.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.de.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.en.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.es.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.it.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml | 2 ++ src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml | 2 ++ .../ImportBundle/Resources/views/Import/_workerEnabled.html.twig | 8 ++++++++ src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig | 4 +++- .../ImportBundle/Resources/views/Readability/index.html.twig | 2 ++ .../ImportBundle/Resources/views/WallabagV1/index.html.twig | 2 ++ 15 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index 073dee28..17dbd53a 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -346,6 +346,8 @@ import: # page_title: 'Import > Readability' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index 4cfd240f..1b844859 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -346,6 +346,8 @@ import: page_title: 'Aus Readability importieren' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: page_title: 'Entwickler' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 42374b40..d0e6aaa4 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -346,6 +346,8 @@ import: page_title: 'Import > Readability' description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index ee84cc62..8aa44582 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -346,6 +346,8 @@ import: page_title: 'Importar > Readability' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: page_title: 'Promotor' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index e9af1e8d..cc373a62 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -346,6 +346,8 @@ import: page_title: 'درون‌ریزی > Readability' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index 402cdf4a..3207b7f9 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -346,6 +346,8 @@ import: page_title: 'Importer > Readability' description: 'Cet outil va importer toutes vos données de Readability. Sur la page des outils (https://www.readability.com/tools/), cliquez sur "Export your data" dans la section "Data Export". Vous allez recevoir un email avec un lien pour télécharger le json.' how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer." + worker: + enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :" developer: page_title: 'Développeur' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 3aee4816..2df7d70e 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -345,6 +345,8 @@ import: page_title: 'Importa da > Readability' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: page_title: 'Sviluppatori' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index a7868fc1..cf869942 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -346,6 +346,8 @@ import: page_title: 'Importer > Readability' description: "Aquesta aisina importarà totas vòstres articles de Readability. Sus la pagina de l'aisina (https://www.readability.com/tools/), clicatz sus \"Export your data\" dins la seccion \"Data Export\". Recebretz un corrièl per telecargar un json (qu'acaba pas amb un .json de fach)." how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: page_title: 'Desvolopador' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index da50cd4c..ab0b680b 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -346,6 +346,8 @@ import: page_title: 'Import > Readability' description: 'Ten importer, zaimportuje wszystkie twoje artykuły z Readability. Na stronie narzędzi (https://www.readability.com/tools/), kliknij na "Export your data" w sekcji "Data Export". Otrzymach email z plikiem JSON (plik nie będzie zawierał rozszerzenia .json).' how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: page_title: 'Deweloper' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index f41609d0..654329f8 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -346,6 +346,8 @@ import: # page_title: 'Import > Readability' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 6dfbfa89..9751aa28 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -346,6 +346,8 @@ import: page_title: 'İçe Aktar > Readability' # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + worker: + # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig new file mode 100644 index 00000000..2390a41f --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig @@ -0,0 +1,8 @@ +{% set redis = craue_setting('import_with_redis') %} +{% set rabbit = craue_setting('import_with_rabbitmq') %} + +{% if redis or rabbit %} +
+ {{ 'import.worker.enabled'|trans }} {% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %} +
+{% endif %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig index 401dbd33..aa5941b7 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig @@ -6,8 +6,10 @@
+ {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% if not has_consumer_key %} -
+
{{ 'import.pocket.config_missing.description'|trans }} {% if is_granted('ROLE_SUPER_ADMIN') %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig index f527d309..74653b0f 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} +
{{ import.description|trans }}

{{ 'import.readability.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig index 13e24c8c..0b19bc34 100644 --- a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} +
{{ import.description|trans }}

{{ 'import.wallabag_v1.how_to'|trans }}

-- cgit v1.2.3 From 7d862f83b95d24b4f081d73ca7b0bdf4435ae008 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 20:23:17 +0200 Subject: Re-facto EntryConsumer Using an abstract method allow to share code but also can be used it we add a new broker in the future --- .../ImportBundle/Consumer/AMPQEntryConsumer.php | 57 +---------------- .../ImportBundle/Consumer/AbstractConsumer.php | 74 ++++++++++++++++++++++ .../ImportBundle/Consumer/RedisEntryConsumer.php | 59 +---------------- 3 files changed, 78 insertions(+), 112 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php index 39bb5375..d95a011d 100644 --- a/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php +++ b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php @@ -2,69 +2,16 @@ namespace Wallabag\ImportBundle\Consumer; -use Doctrine\ORM\EntityManager; use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface; use PhpAmqpLib\Message\AMQPMessage; -use Wallabag\ImportBundle\Import\AbstractImport; -use Wallabag\UserBundle\Repository\UserRepository; -use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\CoreBundle\Entity\Tag; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -class AMPQEntryConsumer implements ConsumerInterface +class AMPQEntryConsumer extends AbstractConsumer implements ConsumerInterface { - private $em; - private $userRepository; - private $import; - private $logger; - - public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null) - { - $this->em = $em; - $this->userRepository = $userRepository; - $this->import = $import; - $this->logger = $logger ?: new NullLogger(); - } - /** * {@inheritdoc} */ public function execute(AMQPMessage $msg) { - $storedEntry = json_decode($msg->body, true); - - $user = $this->userRepository->find($storedEntry['userId']); - - // no user? Drop message - if (null === $user) { - $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); - - return; - } - - $this->import->setUser($user); - - $entry = $this->import->parseEntry($storedEntry); - - if (null === $entry) { - $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); - - return; - } - - try { - $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); - } catch (\Exception $e) { - $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); - - return; - } - - $this->logger->info('Content with url ('.$entry->getUrl().') imported !'); + return $this->handleMessage($msg->body); } } diff --git a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php new file mode 100644 index 00000000..2b85ad76 --- /dev/null +++ b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php @@ -0,0 +1,74 @@ +em = $em; + $this->userRepository = $userRepository; + $this->import = $import; + $this->logger = $logger ?: new NullLogger(); + } + + /** + * Handle a message and save it. + * + * @param string $body Message from the queue (in json) + * + * @return bool + */ + protected function handleMessage($body) + { + $storedEntry = json_decode($body, true); + + $user = $this->userRepository->find($storedEntry['userId']); + + // no user? Drop message + if (null === $user) { + $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); + + return false; + } + + $this->import->setUser($user); + + $entry = $this->import->parseEntry($storedEntry); + + if (null === $entry) { + $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); + + return false; + } + + try { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } catch (\Exception $e) { + $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); + + return false; + } + + $this->logger->info('Content with url imported! ('.$entry->getUrl().')'); + + return true; + } +} diff --git a/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php index 38665b01..450b71ff 100644 --- a/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php +++ b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php @@ -3,29 +3,9 @@ namespace Wallabag\ImportBundle\Consumer; use Simpleue\Job\Job; -use Doctrine\ORM\EntityManager; -use Wallabag\ImportBundle\Import\AbstractImport; -use Wallabag\UserBundle\Repository\UserRepository; -use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\CoreBundle\Entity\Tag; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -class RedisEntryConsumer implements Job +class RedisEntryConsumer extends AbstractConsumer implements Job { - private $em; - private $userRepository; - private $import; - private $logger; - - public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null) - { - $this->em = $em; - $this->userRepository = $userRepository; - $this->import = $import; - $this->logger = $logger ?: new NullLogger(); - } - /** * Handle one message by one message. * @@ -35,42 +15,7 @@ class RedisEntryConsumer implements Job */ public function manage($job) { - $storedEntry = json_decode($job, true); - - $user = $this->userRepository->find($storedEntry['userId']); - - // no user? Drop message - if (null === $user) { - $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]); - - return false; - } - - $this->import->setUser($user); - - $entry = $this->import->parseEntry($storedEntry); - - if (null === $entry) { - $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); - - return false; - } - - try { - $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); - } catch (\Exception $e) { - $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]); - - return false; - } - - $this->logger->info('Content with url ('.$entry->getUrl().') imported !'); - - return true; + return $this->handleMessage($job); } /** -- cgit v1.2.3 From 015c7a8359c950f9621b38b11c3973860a981da8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 20:24:04 +0200 Subject: Add more tests And ability to define how many messages can be hanle by the redis worker before stopping (usefull for tests) --- .../ImportBundle/Command/RedisWorkerCommand.php | 5 +- .../Controller/ReadabilityController.php | 2 +- .../ImportBundle/Controller/WallabagController.php | 2 +- .../ImportBundle/Form/Type/UploadImportType.php | 1 + .../Command/RedisWorkerCommandTest.php | 74 ++++++++++++++++++++++ .../Consumer/RedisEntryConsumerTest.php | 1 + .../Controller/ReadabilityControllerTest.php | 17 +++++ .../Controller/WallabagV1ControllerTest.php | 17 +++++ .../Controller/WallabagV2ControllerTest.php | 17 +++++ 9 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 tests/Wallabag/ImportBundle/Command/RedisWorkerCommandTest.php diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php index 85c5a903..5f90e00f 100644 --- a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php +++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php @@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Command; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; use Symfony\Component\Config\Definition\Exception\Exception; use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Simpleue\Worker\QueueWorker; @@ -17,6 +18,7 @@ class RedisWorkerCommand extends ContainerAwareCommand ->setName('wallabag:import:redis-worker') ->setDescription('Launch Redis worker') ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability') + ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false) ; } @@ -33,7 +35,8 @@ class RedisWorkerCommand extends ContainerAwareCommand $worker = new QueueWorker( $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName), - $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName) + $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName), + $input->getOption('maxIterations') ); $worker->start(); diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index 61243042..8775e8a3 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -31,7 +31,7 @@ class ReadabilityController extends Controller $markAsRead = $form->get('mark_as_read')->getData(); $name = 'readability_'.$this->getUser()->getId().'.json'; - if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $readability ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php index 1e6114c5..3c2dd6d1 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagController.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php @@ -45,7 +45,7 @@ abstract class WallabagController extends Controller $markAsRead = $form->get('mark_as_read')->getData(); $name = $this->getUser()->getId().'.json'; - if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $wallabag ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) diff --git a/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php b/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php index 92a167d9..f50424c1 100644 --- a/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php +++ b/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php @@ -15,6 +15,7 @@ class UploadImportType extends AbstractType $builder ->add('file', FileType::class, [ 'label' => 'import.form.file_label', + 'required' => true, ]) ->add('mark_as_read', CheckboxType::class, [ 'label' => 'import.form.mark_as_read_label', diff --git a/tests/Wallabag/ImportBundle/Command/RedisWorkerCommandTest.php b/tests/Wallabag/ImportBundle/Command/RedisWorkerCommandTest.php new file mode 100644 index 00000000..74952847 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Command/RedisWorkerCommandTest.php @@ -0,0 +1,74 @@ +getClient()->getKernel()); + $application->add(new RedisWorkerCommand()); + + $command = $application->find('wallabag:import:redis-worker'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + ]); + } + + /** + * @expectedException Symfony\Component\Config\Definition\Exception\Exception + * @expectedExceptionMessage No queue or consumer found for service name + */ + public function testRunRedisWorkerCommandWithBadService() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new RedisWorkerCommand()); + + $command = $application->find('wallabag:import:redis-worker'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'serviceName' => 'YOMONSERVICE', + ]); + } + + public function testRunRedisWorkerCommand() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new RedisWorkerCommand()); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $application->getKernel()->getContainer()->set('wallabag_core.redis.client', $redisMock); + + // put a fake message in the queue so the worker will stop after reading that message + // instead of waiting for others + $redisMock->lpush('wallabag.import.readability', '{}'); + + $command = $application->find('wallabag:import:redis-worker'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'serviceName' => 'readability', + '--maxIterations' => 1, + ]); + + $this->assertContains('Worker started at', $tester->getDisplay()); + $this->assertContains('Waiting for message', $tester->getDisplay()); + } +} diff --git a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php index 0ce7ce49..5e8ee41d 100644 --- a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php +++ b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php @@ -220,5 +220,6 @@ JSON; $res = $consumer->manage($body); $this->assertFalse($res); + $this->assertFalse($consumer->isStopJob($body)); } } diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index 69635382..fb835f62 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -51,6 +51,23 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } + public function testImportReadabilityBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/readability'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + public function testImportReadabilityWithFile() { $this->logInAs('admin'); diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php index 933ddd6c..f1113365 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php @@ -51,6 +51,23 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } + public function testImportWallabagBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/wallabag-v1'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + public function testImportWallabagWithFile() { $this->logInAs('admin'); diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php index 36e5221d..b20226ad 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php @@ -51,6 +51,23 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } + public function testImportWallabagBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/wallabag-v2'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + public function testImportWallabagWithFile() { $this->logInAs('admin'); -- cgit v1.2.3 From 03e078d060db410fe56b978cdf45d8eec5e177cb Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 20:43:16 +0200 Subject: Update docs about worker --- docs/en/developer/rabbitmq.rst | 26 +++++++++++++++--- docs/en/developer/redis.rst | 62 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 docs/en/developer/redis.rst diff --git a/docs/en/developer/rabbitmq.rst b/docs/en/developer/rabbitmq.rst index a17e6e4d..8cee45fb 100644 --- a/docs/en/developer/rabbitmq.rst +++ b/docs/en/developer/rabbitmq.rst @@ -1,7 +1,7 @@ Install RabbitMQ for asynchronous tasks ======================================= -In order to launch asynchronous tasks (useful for huge imports for example), we use RabbitMQ. +In order to launch asynchronous tasks (useful for huge imports for example), we can use RabbitMQ. Requirements ------------ @@ -37,13 +37,31 @@ Stop RabbitMQ Configure RabbitMQ in wallabag ------------------------------ -Edit your ``parameters.yml`` file to edit RabbitMQ configuration. +Edit your ``parameters.yml`` file to edit RabbitMQ configuration. The default one should be ok: + +.. code:: yaml + + rabbitmq_host: localhost + rabbitmq_port: 5672 + rabbitmq_user: guest + rabbitmq_password: guest + Launch RabbitMQ consumer ------------------------ -Put this command in a cron job: +Depending on which service you want to import from you need to enable one (or many if you want to support many) cron job: .. code:: bash - bin/console rabbitmq:consumer entries -w \ No newline at end of file + # for Pocket import + bin/console rabbitmq:consumer import_pocket -w + + # for Readbility import + bin/console rabbitmq:consumer import_readability -w + + # for wallabag v1 import + bin/console rabbitmq:consumer import_wallabag_v1 -w + + # for wallabag v2 import + bin/console rabbitmq:consumer import_wallabag_v2 -w diff --git a/docs/en/developer/redis.rst b/docs/en/developer/redis.rst new file mode 100644 index 00000000..5748e260 --- /dev/null +++ b/docs/en/developer/redis.rst @@ -0,0 +1,62 @@ +Install Redis for asynchronous tasks +======================================= + +In order to launch asynchronous tasks (useful for huge imports for example), we can use Redis. + +Requirements +------------ + +You need to have Redis installed on your server. + +Installation +~~~~~~~~~~~~ + +.. code:: bash + + apt-get install redis-server + +Launch +~~~~~~ + +The server might be already running after installing, if not you can launch it using: + +.. code:: bash + + redis-server + + +Configure Redis in wallabag +--------------------------- + +Edit your ``parameters.yml`` file to edit Redis configuration. The default one should be ok: + +.. code:: yaml + + redis_host: localhost + redis_port: 6379 + + +Launch Redis consumer +------------------------ + +Depending on which service you want to import from you need to enable one (or many if you want to support many) cron job: + +.. code:: bash + + # for Pocket import + bin/console wallabag:import:redis-worker pocket -vv >> /path/to/wallabag/var/logs/redis-pocket.log + + # for Readbility import + bin/console wallabag:import:redis-worker readability -vv >> /path/to/wallabag/var/logs/redis-readability.log + + # for wallabag v1 import + bin/console wallabag:import:redis-worker wallabag_v1 -vv >> /path/to/wallabag/var/logs/redis-wallabag_v1.log + + # for wallabag v2 import + bin/console wallabag:import:redis-worker wallabag_v2 -vv >> /path/to/wallabag/var/logs/redis-wallabag_v2.log + +If you want to launch the import only for some messages and not all, you can specify this number (here 12) and the worker will stop right after the 12th message : + +.. code:: bash + + bin/console wallabag:import:redis-worker pocket -vv --maxIterations=12 -- cgit v1.2.3 From ebf5e5087d2f79ece42a660ee7bddaa3ff3ebe1e Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 21:40:08 +0200 Subject: Add tests on ImportCommand --- .../ImportBundle/Command/ImportCommand.php | 6 +- .../ImportBundle/Command/ImportCommandTest.php | 86 ++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/Wallabag/ImportBundle/Command/ImportCommandTest.php diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index a4aa8531..20ecc6e1 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -26,6 +26,10 @@ class ImportCommand extends ContainerAwareCommand { $output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---'); + if (!file_exists($input->getArgument('filepath'))) { + throw new Exception(sprintf('File "%s" not found', $input->getArgument('filepath'))); + } + $em = $this->getContainer()->get('doctrine')->getManager(); // Turning off doctrine default logs queries for saving memory $em->getConnection()->getConfiguration()->setSQLLogger(null); @@ -43,9 +47,9 @@ class ImportCommand extends ContainerAwareCommand } $wallabag->setMarkAsRead($input->getOption('markAsRead')); + $wallabag->setUser($user); $res = $wallabag - ->setUser($user) ->setFilepath($input->getArgument('filepath')) ->import(); diff --git a/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php b/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php new file mode 100644 index 00000000..eb7fce79 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Command/ImportCommandTest.php @@ -0,0 +1,86 @@ +getClient()->getKernel()); + $application->add(new ImportCommand()); + + $command = $application->find('wallabag:import'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + ]); + } + + /** + * @expectedException Symfony\Component\Config\Definition\Exception\Exception + * @expectedExceptionMessage not found + */ + public function testRunImportCommandWithoutFilepath() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new ImportCommand()); + + $command = $application->find('wallabag:import'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'userId' => 1, + 'filepath' => 1, + ]); + } + + /** + * @expectedException Symfony\Component\Config\Definition\Exception\Exception + * @expectedExceptionMessage User with id + */ + public function testRunImportCommandWithoutUserId() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new ImportCommand()); + + $command = $application->find('wallabag:import'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'userId' => 0, + 'filepath' => './', + ]); + } + + public function testRunImportCommand() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new ImportCommand()); + + $command = $application->find('wallabag:import'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'userId' => 1, + 'filepath' => $application->getKernel()->getContainer()->getParameter('kernel.root_dir').'/../tests/Wallabag/ImportBundle/fixtures/wallabag-v2-read.json', + '--importer' => 'v2', + ]); + + $this->assertContains('imported', $tester->getDisplay()); + $this->assertContains('already saved', $tester->getDisplay()); + } +} -- cgit v1.2.3 From 886d47973466b5516a633b0e6dd43269082f1676 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 11 Sep 2016 23:57:27 +0200 Subject: Fix tests --- tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php index b20226ad..ca20c85b 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php @@ -99,9 +99,9 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $this->getLoggedInUserId() ); - $this->assertEmpty($content->getMimetype()); - $this->assertEmpty($content->getPreviewPicture()); - $this->assertEmpty($content->getLanguage()); + $this->assertNotEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertNotEmpty($content->getLanguage()); $this->assertEquals(0, count($content->getTags())); $content = $client->getContainer() -- cgit v1.2.3 From 5d002e9bdfd814e77e5842fe6c18d14f74493726 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 12 Sep 2016 09:07:53 +0200 Subject: Update import.worker.enabled translation --- src/Wallabag/CoreBundle/Resources/translations/messages.da.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.de.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.en.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.es.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml | 6 +++--- src/Wallabag/CoreBundle/Resources/translations/messages.it.yml | 6 +++--- src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml | 2 +- src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml | 6 +++--- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index 17dbd53a..3dad0b8f 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -347,7 +347,7 @@ import: # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index 1b844859..c0a674e3 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -347,7 +347,7 @@ import: # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: page_title: 'Entwickler' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index d0e6aaa4..4104f4d4 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -347,7 +347,7 @@ import: description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index 8aa44582..e5604fda 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -347,7 +347,7 @@ import: # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: page_title: 'Promotor' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index cc373a62..63464d94 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -344,10 +344,10 @@ import: description: 'این برنامه همهٔ داده‌های شما را در نسخهٔ ۲ wallabag درون‌ریزی می‌کند. به بخش «همهٔ مقاله‌ها» بروید و در بخش «برون‌ریزی» روی "JSON" کلیک کنید. با این کار شما پرونده‌ای به شکل "All articles.json" دریافت خواهید کرد.' readability: page_title: 'درون‌ریزی > Readability' - # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' - # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' + # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 2df7d70e..5be376c8 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -343,10 +343,10 @@ import: description: 'Questo importatore copierà tutti i tuoi dati da un wallabag v2. Vai in "Tutti i contenuti", e, nella sidebar di esportazione, clicca su "JSON". Otterrai un file "Tutti i contenuti.json".' readability: page_title: 'Importa da > Readability' - # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' - # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' + # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: page_title: 'Sviluppatori' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index cf869942..3778b868 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -347,7 +347,7 @@ import: description: "Aquesta aisina importarà totas vòstres articles de Readability. Sus la pagina de l'aisina (https://www.readability.com/tools/), clicatz sus \"Export your data\" dins la seccion \"Data Export\". Recebretz un corrièl per telecargar un json (qu'acaba pas amb un .json de fach)." how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: page_title: 'Desvolopador' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index ab0b680b..8b536679 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -347,7 +347,7 @@ import: description: 'Ten importer, zaimportuje wszystkie twoje artykuły z Readability. Na stronie narzędzi (https://www.readability.com/tools/), kliknij na "Export your data" w sekcji "Data Export". Otrzymach email z plikiem JSON (plik nie będzie zawierał rozszerzenia .json).' how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: page_title: 'Deweloper' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 654329f8..3f330da4 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -347,7 +347,7 @@ import: # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 9751aa28..5216ebe5 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -344,10 +344,10 @@ import: # description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.' readability: page_title: 'İçe Aktar > Readability' - # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' - # how_to: 'Please select your Readability export and click on the below button to upload and import it.' + # description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact).' + # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: - # enabled: "Imports are asynchronous. Once the import is submitted an external worker will handle message one by one. The current service is:" + # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" developer: # page_title: 'Developer' -- cgit v1.2.3 From e01a3c98d6908e95121b5ade0161f40af1b05ca6 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 20:31:32 +0200 Subject: Display how many messages are queue - update the docker-composer to add Redis - add migrations --- app/DoctrineMigrations/Version20160911214952.php | 42 +++++++++++++++++ docker-compose.yml | 9 ++++ .../views/themes/material/layout.html.twig | 2 + .../ImportBundle/Controller/ImportController.php | 54 ++++++++++++++++++++++ .../Resources/views/Import/check_queue.html.twig | 11 +++++ 5 files changed, 118 insertions(+) create mode 100644 app/DoctrineMigrations/Version20160911214952.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig diff --git a/app/DoctrineMigrations/Version20160911214952.php b/app/DoctrineMigrations/Version20160911214952.php new file mode 100644 index 00000000..35809cec --- /dev/null +++ b/app/DoctrineMigrations/Version20160911214952.php @@ -0,0 +1,42 @@ +container = $container; + } + + private function getTable($tableName) + { + return $this->container->getParameter('database_table_prefix') . $tableName; + } + + /** + * @param Schema $schema + */ + public function up(Schema $schema) + { + $this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_redis\', \'0\', \'import\')'); + $this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_rabbitmq\', \'0\', \'import\')'); + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + } +} diff --git a/docker-compose.yml b/docker-compose.yml index a255b070..27f0f14f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: links: - php:php command: nginx -c /nginx.conf + php: build: context: docker/php @@ -30,6 +31,7 @@ services: # If all DBMS are commented out, sqlite will be used as default # - ./docker/postgres/env # - ./docker/mariadb/env + #postgres: # image: postgres:9 # ports: @@ -38,6 +40,7 @@ services: # - ./docker/data/pgsql:/var/lib/postgresql/data # env_file: # - ./docker/postgres/env + #mariadb: # image: mariadb:10 # ports: @@ -46,7 +49,13 @@ services: # - ./docker/data/mariadb:/var/lib/mysql # env_file: # - ./docker/mariadb/env + rabbitmq: image: rabbitmq:3-management ports: - "15672:15672" + + rabbitmq: + image: redis + ports: + - "6379:6379" diff --git a/src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig b/src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig index b70198da..23938b21 100644 --- a/src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig +++ b/src/Wallabag/CoreBundle/Resources/views/themes/material/layout.html.twig @@ -19,6 +19,8 @@ Materialize.toast('{{ flashMessage|trans }}', 4000); {% endfor %} + + {{ render(controller("WallabagImportBundle:Import:checkQueue")) }} {% endblock %} {% block menu %} diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index c1486e38..d670746c 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -16,4 +16,58 @@ class ImportController extends Controller 'imports' => $this->get('wallabag_import.chain')->getAll(), ]); } + + /** + * Display how many messages are queue (both in Redis and RabbitMQ). + */ + public function checkQueueAction() + { + $nbRedisMessages = null; + $nbRabbitMessages = null; + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket') + + $this->getTotalMessageInRabbitQueue('readability') + + $this->getTotalMessageInRabbitQueue('wallabag_v1') + + $this->getTotalMessageInRabbitQueue('wallabag_v2') + ; + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $redis = $this->get('wallabag_core.redis.client'); + + $nbRedisMessages = $redis->llen('wallabag.import.pocket') + + $redis->llen('wallabag.import.readability') + + $redis->llen('wallabag.import.wallabag_v1') + + $redis->llen('wallabag.import.wallabag_v2') + ; + } + + return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [ + 'nbRedisMessages' => $nbRedisMessages, + 'nbRabbitMessages' => $nbRabbitMessages, + ]); + } + + /** + * Count message in RabbitMQ queue. + * It get one message without acking it (so it'll stay in the queue) + * which will include the total of *other* messages in the queue. + * Adding one to that messages will result in the full total message. + * + * @param string $importService The import service related: pocket, readability, wallabag_v1 or wallabag_v2 + * + * @return int + */ + private function getTotalMessageInRabbitQueue($importService) + { + $message = $this + ->get('old_sound_rabbit_mq.import_'.$importService.'_consumer') + ->getChannel() + ->basic_get('wallabag.import.'.$importService); + + if (null === $message) { + return 0; + } + + return $message->delivery_info['message_count'] + 1; + } } diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig new file mode 100644 index 00000000..7168ea35 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig @@ -0,0 +1,11 @@ +{% if nbRedisMessages > 0 %} + +{% endif %} + +{% if nbRabbitMessages > 0 %} + +{% endif %} -- cgit v1.2.3 From c80cc01afa315dcfa38b2a01c5b05d4516659c24 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 21:09:05 +0200 Subject: Change flash message for queued articles --- .../CoreBundle/Resources/translations/messages.da.yml | 1 + .../CoreBundle/Resources/translations/messages.de.yml | 1 + .../CoreBundle/Resources/translations/messages.en.yml | 1 + .../CoreBundle/Resources/translations/messages.es.yml | 7 ++++--- .../CoreBundle/Resources/translations/messages.fa.yml | 1 + .../CoreBundle/Resources/translations/messages.fr.yml | 1 + .../CoreBundle/Resources/translations/messages.it.yml | 1 + .../CoreBundle/Resources/translations/messages.oc.yml | 1 + .../CoreBundle/Resources/translations/messages.pl.yml | 1 + .../CoreBundle/Resources/translations/messages.ro.yml | 1 + .../CoreBundle/Resources/translations/messages.tr.yml | 1 + src/Wallabag/ImportBundle/Controller/PocketController.php | 6 ++++++ .../ImportBundle/Controller/ReadabilityController.php | 6 ++++++ .../ImportBundle/Controller/WallabagController.php | 6 ++++++ src/Wallabag/ImportBundle/Import/AbstractImport.php | 15 ++++++++++++++- src/Wallabag/ImportBundle/Import/PocketImport.php | 11 ----------- src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 11 ----------- src/Wallabag/ImportBundle/Import/WallabagImport.php | 11 ----------- tests/Wallabag/ImportBundle/Import/PocketImportTest.php | 10 +++++----- .../ImportBundle/Import/ReadabilityImportTest.php | 8 ++++---- .../Wallabag/ImportBundle/Import/WallabagV1ImportTest.php | 8 ++++---- .../Wallabag/ImportBundle/Import/WallabagV2ImportTest.php | 12 ++++++------ 22 files changed, 65 insertions(+), 56 deletions(-) diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index 3dad0b8f..4e5ed9ae 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -430,6 +430,7 @@ flashes: # failed: 'Import failed, please try again.' # failed_on_file: 'Error while processing import. Please verify your import file.' # summary: 'Import summary: %imported% imported, %skipped% already saved.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: # client_created: 'New client created.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index c0a674e3..107187bf 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -430,6 +430,7 @@ flashes: failed: 'Import fehlgeschlagen, bitte erneut probieren.' failed_on_file: 'Fehler während des Imports. Bitte überprüfe deine Import-Datei.' summary: 'Import-Zusammenfassung: %imported% importiert, %skipped% bereits gespeichert.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: client_created: 'Neuer Client erstellt.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 4104f4d4..ac60295f 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -432,6 +432,7 @@ flashes: failed: 'Import failed, please try again.' failed_on_file: 'Error while processing import. Please verify your import file.' summary: 'Import summary: %imported% imported, %skipped% already saved.' + summary_with_queue: 'Import summary: %queued% queued.' developer: notice: client_created: 'New client %name% created.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index e5604fda..1082e6de 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -427,9 +427,10 @@ flashes: tag_added: 'Etiqueta añadida' import: notice: - failed: 'Importación reprobada, por favor inténtelo de nuevo.' - failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.' - summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.' + failed: 'Importación reprobada, por favor inténtelo de nuevo.' + failed_on_file: 'Se ocurre un error por procesar importación. Por favor verifique su archivo importado.' + summary: 'Resúmen importado: %importado% importado, %saltados% ya guardado.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: client_created: 'Nuevo cliente creado.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 63464d94..78317a9c 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -430,6 +430,7 @@ flashes: failed: 'درون‌ریزی شکست خورد. لطفاً دوباره تلاش کنید.' failed_on_file: 'خطا هنگام پردازش پروندهٔ ورودی. آیا پروندهٔ درون‌ریزی شده سالم است؟' summary: 'گزارش درون‌ریزی: %imported% وارد شد, %skipped% از قبل ذخیره شده بود.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: # client_created: 'New client created.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index 3207b7f9..50515d80 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -432,6 +432,7 @@ flashes: failed: "L'import a échoué, veuillez ré-essayer" failed_on_file: "Erreur lors du traitement de l'import. Vérifier votre fichier." summary: "Rapport d'import: %imported% importés, %skipped% déjà présent." + summary_with_queue: "Rapport d'import: %queued% en cours de traitement." developer: notice: client_created: 'Nouveau client %name% créé' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 5be376c8..5badeffc 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -429,6 +429,7 @@ flashes: failed: 'Importazione fallita, riprova.' failed_on_file: 'Errore durante la processazione dei dati da importare. Verifica il tuo file di import.' summary: 'Sommario di importazione: %imported% importati, %skipped% già salvati.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: client_created: 'Nuovo client creato.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index 3778b868..2383886c 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -430,6 +430,7 @@ flashes: failed: "L'importacion a fracassat, mercés de tornar ensajar" failed_on_file: "Errorr pendent du tractament de l'import. Mercés de verificar vòstre fichièr." summary: "Rapòrt d'import: %imported% importats, %skipped% ja presents." + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: client_created: 'Novèl client creat' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index 8b536679..40cd2b71 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -430,6 +430,7 @@ flashes: failed: 'Nieudany import, prosimy spróbować ponownie.' failed_on_file: 'Błąd podczas ptrzetwarzania pliku. Sprawdż swój importowany plik.' summary: 'Podsumowanie importu: %imported% zaimportowane, %skipped% już zapisane.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: client_created: 'Nowy klient utworzony.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 3f330da4..b18eab9e 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -430,6 +430,7 @@ flashes: # failed: 'Import failed, please try again.' # failed_on_file: 'Error while processing import. Please verify your import file.' # summary: 'Import summary: %imported% imported, %skipped% already saved.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: # client_created: 'New client created.' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 5216ebe5..82fa93af 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -430,6 +430,7 @@ flashes: # failed: 'Import failed, please try again.' # failed_on_file: 'Error while processing import. Please verify your import file.' # summary: 'Import summary: %imported% imported, %skipped% already saved.' + # summary_with_queue: 'Import summary: %queued% queued.' developer: notice: # client_created: 'New client created.' diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 3d555717..1f92c182 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -102,6 +102,12 @@ class PocketController extends Controller '%imported%' => $summary['imported'], '%skipped%' => $summary['skipped'], ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } } $this->get('session')->getFlashBag()->add( diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php index 8775e8a3..d00e22c2 100644 --- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -46,6 +46,12 @@ class ReadabilityController extends Controller '%skipped%' => $summary['skipped'], ]); + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); } diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php index 3c2dd6d1..9c0cde80 100644 --- a/src/Wallabag/ImportBundle/Controller/WallabagController.php +++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php @@ -60,6 +60,12 @@ abstract class WallabagController extends Controller '%skipped%' => $summary['skipped'], ]); + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); } diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 4cd8e846..2af0e69b 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -21,6 +21,7 @@ abstract class AbstractImport implements ImportInterface protected $markAsRead; protected $skippedEntries = 0; protected $importedEntries = 0; + protected $queuedEntries = 0; public function __construct(EntityManager $em, ContentProxy $contentProxy) { @@ -145,12 +146,24 @@ abstract class AbstractImport implements ImportInterface $importedEntry = $this->setEntryAsRead($importedEntry); } - ++$this->importedEntries; + ++$this->queuedEntries; $this->producer->publish(json_encode($importedEntry)); } } + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + 'queued' => $this->queuedEntries, + ]; + } + /** * Parse one entry. * diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index fe39d33f..cc6faf1f 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -172,17 +172,6 @@ class PocketImport extends AbstractImport return true; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * Set the Guzzle client. * diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index 8f080d38..b852f8f0 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -44,17 +44,6 @@ class ReadabilityImport extends AbstractImport return $this; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 8e50b135..969a6a04 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -72,17 +72,6 @@ abstract class WallabagImport extends AbstractImport return true; } - /** - * {@inheritdoc} - */ - public function getSummary() - { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; - } - /** * Set file path to the json file. * diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 425fa321..a3f68892 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -270,7 +270,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $res = $pocketImport->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 1, 'imported' => 1], $pocketImport->getSummary()); + $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $pocketImport->getSummary()); } /** @@ -365,7 +365,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $res = $pocketImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 2], $pocketImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $pocketImport->getSummary()); } /** @@ -451,7 +451,7 @@ JSON; $res = $pocketImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $pocketImport->getSummary()); } /** @@ -530,7 +530,7 @@ JSON; $res = $pocketImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 1], $pocketImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 1], $pocketImport->getSummary()); $this->assertNotEmpty($redisMock->lpop('pocket')); } @@ -607,6 +607,6 @@ JSON; $res = $pocketImport->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 1, 'imported' => 0], $pocketImport->getSummary()); + $this->assertEquals(['skipped' => 1, 'imported' => 0, 'queued' => 0], $pocketImport->getSummary()); } } diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php index 67917154..d98cd486 100644 --- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php @@ -82,7 +82,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $res = $readabilityImport->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 24], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 24, 'queued' => 0], $readabilityImport->getSummary()); } public function testImportAndMarkAllAsRead() @@ -120,7 +120,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $this->assertTrue($res); - $this->assertEquals(['skipped' => 1, 'imported' => 1], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $readabilityImport->getSummary()); } public function testImportWithRabbit() @@ -160,7 +160,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $res = $readabilityImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 24], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $readabilityImport->getSummary()); } public function testImportWithRedis() @@ -198,7 +198,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase $res = $readabilityImport->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 24], $readabilityImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $readabilityImport->getSummary()); $this->assertNotEmpty($redisMock->lpop('readability')); } diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php index b43682cd..5ab4ad00 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php @@ -82,7 +82,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV1Import->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 1, 'imported' => 3], $wallabagV1Import->getSummary()); + $this->assertEquals(['skipped' => 1, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary()); } public function testImportAndMarkAllAsRead() @@ -120,7 +120,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 3], $wallabagV1Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 3, 'queued' => 0], $wallabagV1Import->getSummary()); } public function testImportWithRabbit() @@ -160,7 +160,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV1Import->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary()); } public function testImportWithRedis() @@ -198,7 +198,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV1Import->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 4], $wallabagV1Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $wallabagV1Import->getSummary()); $this->assertNotEmpty($redisMock->lpop('wallabag_v1')); } diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php index 18998b35..b4017f72 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php @@ -78,7 +78,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV2Import->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 22, 'imported' => 2], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary()); } public function testImportAndMarkAllAsRead() @@ -116,7 +116,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 2], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary()); } public function testImportWithRabbit() @@ -152,7 +152,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV2Import->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary()); } public function testImportWithRedis() @@ -186,7 +186,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV2Import->setMarkAsRead(true)->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 0, 'imported' => 24], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 24], $wallabagV2Import->getSummary()); $this->assertNotEmpty($redisMock->lpop('wallabag_v2')); } @@ -227,7 +227,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV2Import->import(); $this->assertFalse($res); - $this->assertEquals(['skipped' => 0, 'imported' => 0], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 0], $wallabagV2Import->getSummary()); } public function testImportWithExceptionFromGraby() @@ -256,6 +256,6 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV2Import->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 24, 'imported' => 0], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 24, 'imported' => 0, 'queued' => 0], $wallabagV2Import->getSummary()); } } -- cgit v1.2.3 From 47d7c682a453a4831c07db926d22461f310642c6 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 22:27:27 +0200 Subject: =?UTF-8?q?Add=20more=20=E2=80=9Creal=E2=80=9D=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controller/ReadabilityControllerTest.php | 39 ++++++++++++++++------ .../Controller/WallabagV1ControllerTest.php | 39 ++++++++++++++++------ .../Controller/WallabagV2ControllerTest.php | 39 ++++++++++++++++------ 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index fb835f62..7b88d891 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -35,37 +35,56 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); } - public function testImportReadabilityWithRedisEnabled() + public function testImportReadabilityBadFile() { $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('import_with_redis', 1); - $crawler = $client->request('GET', '/import/readability'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); - $this->assertEquals(200, $client->getResponse()->getStatusCode()); - $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); - $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $data = [ + 'upload_import_file[file]' => '', + ]; - $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); } - public function testImportReadabilityBadFile() + public function testImportReadabilityWithRedisEnabled() { $this->logInAs('admin'); $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + $crawler = $client->request('GET', '/import/readability'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + $file = new UploadedFile(__DIR__.'/../fixtures/readability.json', 'readability.json'); + $data = [ - 'upload_import_file[file]' => '', + 'upload_import_file[file]' => $file, ]; $client->submit($form, $data); - $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.readability')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportReadabilityWithFile() diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php index f1113365..98e85d45 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV1ControllerTest.php @@ -35,37 +35,56 @@ class WallabagV1ControllerTest extends WallabagCoreTestCase $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); } - public function testImportWallabagWithRedisEnabled() + public function testImportWallabagBadFile() { $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('import_with_redis', 1); - $crawler = $client->request('GET', '/import/wallabag-v1'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); - $this->assertEquals(200, $client->getResponse()->getStatusCode()); - $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); - $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $data = [ + 'upload_import_file[file]' => '', + ]; - $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); } - public function testImportWallabagBadFile() + public function testImportWallabagWithRedisEnabled() { $this->logInAs('admin'); $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + $crawler = $client->request('GET', '/import/wallabag-v1'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + $file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v1.json', 'wallabag-v1.json'); + $data = [ - 'upload_import_file[file]' => '', + 'upload_import_file[file]' => $file, ]; $client->submit($form, $data); - $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.wallabag_v1')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportWallabagWithFile() diff --git a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php index ca20c85b..74d61f9a 100644 --- a/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/WallabagV2ControllerTest.php @@ -35,37 +35,56 @@ class WallabagV2ControllerTest extends WallabagCoreTestCase $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); } - public function testImportWallabagWithRedisEnabled() + public function testImportWallabagBadFile() { $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('import_with_redis', 1); - $crawler = $client->request('GET', '/import/wallabag-v2'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); - $this->assertEquals(200, $client->getResponse()->getStatusCode()); - $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); - $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $data = [ + 'upload_import_file[file]' => '', + ]; - $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); } - public function testImportWallabagBadFile() + public function testImportWallabagWithRedisEnabled() { $this->logInAs('admin'); $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + $crawler = $client->request('GET', '/import/wallabag-v2'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + $file = new UploadedFile(__DIR__.'/../fixtures/wallabag-v2.json', 'wallabag-v2.json'); + $data = [ - 'upload_import_file[file]' => '', + 'upload_import_file[file]' => $file, ]; $client->submit($form, $data); - $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.wallabag_v2')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } public function testImportWallabagWithFile() -- cgit v1.2.3 From d4e366f6eba38c00b2f1660aa548d8c7164f6cf0 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 13 Sep 2016 22:38:05 +0200 Subject: Add missing translations --- app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml | 3 ++- app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml | 2 ++ app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml | 2 ++ 10 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml index 3478d638..905d7b2b 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml @@ -9,6 +9,8 @@ export_json: Aktiver eksport til JSON export_txt: Aktiver eksport til TXT export_xml: Aktiver eksport til XML pocket_consumer_key: Brugers nøgle til Pocket for at importere materialer (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli-URL, hvis tjenesten er aktiv share_diaspora: Aktiver deling til Diaspora share_mail: Aktiver deling med email diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml index f655a27f..eab50ff2 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml @@ -9,6 +9,8 @@ export_json: JSON-Export aktivieren export_txt: TXT-Export aktivieren export_xml: XML-Export aktivieren pocket_consumer_key: Consumer-Key für Pocket, um Inhalte zu importieren (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli-URL, sofern der Service aktiviert ist share_diaspora: Teilen zu Diaspora aktiveren share_mail: Teilen via E-Mail aktiveren diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml index 731e6099..446c9d05 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml @@ -9,7 +9,8 @@ export_json: Enable JSON export export_txt: Enable TXT export export_xml: Enable XML export pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication) -rabbitmq: Enable RabbitMQ to import data asynchronously +import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, if the service is enabled share_diaspora: Enable share to Diaspora share_mail: Enable share by email diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml index 7aac9adf..42f48bf7 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml @@ -9,6 +9,8 @@ export_json: Activar exportación a JSON export_txt: Activar exportación a TXT export_xml: Activar exportación a XML pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, si el servicio está activado share_diaspora: Activar compartir con Diaspora share_mail: Activar compartir con email diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml index 613cf86d..6fc17c98 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml @@ -9,6 +9,8 @@ export_json: فعال‌سازی برون‌سپاری به JSON export_txt: فعال‌سازی برون‌سپاری به TXT export_xml: فعال‌سازی برون‌سپاری به XML pocket_consumer_key: کلید کاربری Pocket برای درون‌ریزی مطالب (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: نشانی Shaarli، اگر فعال بود share_diaspora: فعال‌سازی هم‌رسانی به Diaspora share_mail: فعال‌سازی هم‌رسانی با ایمیل diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml index f21f2439..08351b77 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml @@ -9,6 +9,8 @@ export_json: Activer l'export JSON export_txt: Activer l'export TXT export_xml: Activer l'export XML pocket_consumer_key: Clé d'authentification Pocket pour importer les données (https://getpocket.com/developer/docs/authentication) +import_with_rabbitmq: Activer RabbitMQ pour gérer les imports de façon asynchrone +import_with_redis: Activer Redis pour gérer les imports de façon asynchrone shaarli_url: URL de Shaarli, si le service Shaarli est activé share_diaspora: Activer le partage vers Diaspora share_mail: Activer le partage par email diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml index 1202edd5..0c8a906d 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml @@ -9,6 +9,8 @@ export_json: Abilita esportazione JSON export_txt: Abilita esportazione TXT export_xml: Abilita esportazione XML pocket_consumer_key: Consumer key per Pocket per importare i contenuti (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, se il servizio è abilitato share_diaspora: Abilita la condivisione con Diaspora share_mail: Abilita la condivisione per email diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml index dd91ee49..f578095e 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml @@ -9,6 +9,8 @@ export_json: Activar l'expòrt JSON export_txt: Activar l'expòrt TXT export_xml: Activar l'expòrt XML pocket_consumer_key: Clau d'autentificacion Pocket per importar las donadas (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: URL de Shaarli, se lo servici Shaarli es activat share_diaspora: Activar lo partatge cap a Diaspora share_mail: Activar lo partatge per corrièl diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml index 93b36c8f..351ee2a8 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml @@ -9,6 +9,8 @@ export_json: Włącz eksport do JSON export_txt: Włącz eksport do TXT export_xml: Włącz eksport do XML pocket_consumer_key: Klucz klienta Pocket do importu zawartości (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: Adress URL Shaarli, jeżeli usługa jest włączona share_diaspora: Włącz udostępnianie dla Diaspora share_mail: Włącz udostępnianie przez email diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml index 326c9f90..b9b41ee7 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml @@ -9,6 +9,8 @@ export_json: Permite exportare JSON export_txt: Permite exportare TXT export_xml: Permite exportare XML pocket_consumer_key: Cheie consumator pentru importarea contentului din Pocket (https://getpocket.com/developer/docs/authentication) +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, dacă serviciul este permis share_diaspora: Permite share către Diaspora share_mail: Permite share prin email -- cgit v1.2.3 From a622fbc444232a5a9e41c0047067ae5e94605819 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 14 Sep 2016 09:10:13 +0200 Subject: Fix typo --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 27f0f14f..3c28f2f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: ports: - "15672:15672" - rabbitmq: + redis: image: redis ports: - "6379:6379" -- cgit v1.2.3 From ac87e0db2ac5db90f1b0639a2d31c7098b4eaa20 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 14 Sep 2016 10:17:22 +0200 Subject: AMPQ -> AMQP --- app/config/config.yml | 8 +- .../ImportBundle/Consumer/AMPQEntryConsumer.php | 17 -- .../ImportBundle/Consumer/AMQPEntryConsumer.php | 17 ++ .../ImportBundle/Resources/config/rabbit.yml | 16 +- .../Consumer/AMPQEntryConsumerTest.php | 225 --------------------- .../Consumer/AMQPEntryConsumerTest.php | 225 +++++++++++++++++++++ 6 files changed, 254 insertions(+), 254 deletions(-) delete mode 100644 src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php create mode 100644 src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php delete mode 100644 tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php create mode 100644 tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php diff --git a/app/config/config.yml b/app/config/config.yml index e18a932d..4b869c4f 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -254,7 +254,7 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.pocket' - callback: wallabag_import.consumer.ampq.pocket + callback: wallabag_import.consumer.amqp.pocket import_readability: connection: default exchange_options: @@ -262,7 +262,7 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.readability' - callback: wallabag_import.consumer.ampq.readability + callback: wallabag_import.consumer.amqp.readability import_wallabag_v1: connection: default exchange_options: @@ -270,7 +270,7 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.wallabag_v1' - callback: wallabag_import.consumer.ampq.wallabag_v1 + callback: wallabag_import.consumer.amqp.wallabag_v1 import_wallabag_v2: connection: default exchange_options: @@ -278,4 +278,4 @@ old_sound_rabbit_mq: type: topic queue_options: name: 'wallabag.import.wallabag_v2' - callback: wallabag_import.consumer.ampq.wallabag_v2 + callback: wallabag_import.consumer.amqp.wallabag_v2 diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php deleted file mode 100644 index d95a011d..00000000 --- a/src/Wallabag/ImportBundle/Consumer/AMPQEntryConsumer.php +++ /dev/null @@ -1,17 +0,0 @@ -handleMessage($msg->body); - } -} diff --git a/src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php new file mode 100644 index 00000000..5aaa8c03 --- /dev/null +++ b/src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php @@ -0,0 +1,17 @@ +handleMessage($msg->body); + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index f09dda0d..aa049749 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -1,28 +1,28 @@ # RabbitMQ stuff services: - wallabag_import.consumer.ampq.pocket: - class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + wallabag_import.consumer.amqp.pocket: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" - "@logger" - wallabag_import.consumer.ampq.readability: - class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + wallabag_import.consumer.amqp.readability: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" - "@logger" - wallabag_import.consumer.ampq.wallabag_v1: - class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + wallabag_import.consumer.amqp.wallabag_v1: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" - "@logger" - wallabag_import.consumer.ampq.wallabag_v2: - class: Wallabag\ImportBundle\Consumer\AMPQEntryConsumer + wallabag_import.consumer.amqp.wallabag_v2: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" diff --git a/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php deleted file mode 100644 index b13ade1d..00000000 --- a/tests/Wallabag/ImportBundle/Consumer/AMPQEntryConsumerTest.php +++ /dev/null @@ -1,225 +0,0 @@ -getMockBuilder('Doctrine\ORM\EntityManager') - ->disableOriginalConstructor() - ->getMock(); - - $em - ->expects($this->once()) - ->method('flush'); - - $em - ->expects($this->exactly(2)) - ->method('clear'); - - $body = <<<'JSON' -{ - "item_id": "1402935436", - "resolved_id": "1402935436", - "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", - "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules", - "favorite": "0", - "status": "0", - "time_added": "1473020899", - "time_updated": "1473020899", - "time_read": "0", - "time_favorited": "0", - "sort_id": 0, - "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules", - "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", - "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...", - "is_article": "1", - "is_index": "0", - "has_video": "0", - "has_image": "1", - "word_count": "200", - "tags": { - "ifttt": { - "item_id": "1402935436", - "tag": "ifttt" - }, - "mashable": { - "item_id": "1402935436", - "tag": "mashable" - } - }, - "authors": { - "2484273": { - "item_id": "1402935436", - "author_id": "2484273", - "name": "Adam Rosenberg", - "url": "http://mashable.com/author/adam-rosenberg/" - } - }, - "image": { - "item_id": "1402935436", - "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", - "width": "0", - "height": "0" - }, - "images": { - "1": { - "item_id": "1402935436", - "image_id": "1", - "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", - "width": "0", - "height": "0", - "credit": "Image: Steve Eichner/NameFace/Sipa USA", - "caption": "" - } - }, - "userId": 1 -} -JSON; - - $user = new User(); - $entry = new Entry($user); - - $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') - ->disableOriginalConstructor() - ->getMock(); - - $userRepository - ->expects($this->once()) - ->method('find') - // userId from the body json above - ->with(1) - ->willReturn($user); - - $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') - ->disableOriginalConstructor() - ->getMock(); - - $import - ->expects($this->once()) - ->method('setUser') - ->with($user); - - $import - ->expects($this->once()) - ->method('parseEntry') - ->with(json_decode($body, true)) - ->willReturn($entry); - - $consumer = new AMPQEntryConsumer( - $em, - $userRepository, - $import - ); - - $message = new AMQPMessage($body); - - $consumer->execute($message); - } - - public function testMessageWithBadUser() - { - $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') - ->disableOriginalConstructor() - ->getMock(); - - $em - ->expects($this->never()) - ->method('flush'); - - $em - ->expects($this->never()) - ->method('clear'); - - $body = '{ "userId": 123 }'; - - $user = new User(); - $entry = new Entry($user); - - $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') - ->disableOriginalConstructor() - ->getMock(); - - $userRepository - ->expects($this->once()) - ->method('find') - // userId from the body json above - ->with(123) - ->willReturn(null); - - $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') - ->disableOriginalConstructor() - ->getMock(); - - $consumer = new AMPQEntryConsumer( - $em, - $userRepository, - $import - ); - - $message = new AMQPMessage($body); - - $consumer->execute($message); - } - - public function testMessageWithEntryProcessed() - { - $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') - ->disableOriginalConstructor() - ->getMock(); - - $em - ->expects($this->never()) - ->method('flush'); - - $em - ->expects($this->never()) - ->method('clear'); - - $body = '{ "userId": 123 }'; - - $user = new User(); - - $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') - ->disableOriginalConstructor() - ->getMock(); - - $userRepository - ->expects($this->once()) - ->method('find') - // userId from the body json above - ->with(123) - ->willReturn($user); - - $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') - ->disableOriginalConstructor() - ->getMock(); - - $import - ->expects($this->once()) - ->method('setUser') - ->with($user); - - $import - ->expects($this->once()) - ->method('parseEntry') - ->with(json_decode($body, true)) - ->willReturn(null); - - $consumer = new AMPQEntryConsumer( - $em, - $userRepository, - $import - ); - - $message = new AMQPMessage($body); - - $consumer->execute($message); - } -} diff --git a/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php new file mode 100644 index 00000000..a3263771 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php @@ -0,0 +1,225 @@ +getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->once()) + ->method('flush'); + + $em + ->expects($this->exactly(2)) + ->method('clear'); + + $body = <<<'JSON' +{ + "item_id": "1402935436", + "resolved_id": "1402935436", + "given_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "given_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "favorite": "0", + "status": "0", + "time_added": "1473020899", + "time_updated": "1473020899", + "time_read": "0", + "time_favorited": "0", + "sort_id": 0, + "resolved_title": "Leslie Jones is back on Twitter and her comeback tweet rules", + "resolved_url": "http://mashable.com/2016/09/04/leslie-jones-back-on-twitter-after-hack/?utm_campaign=Mash-Prod-RSS-Feedburner-All-Partial&utm_cid=Mash-Prod-RSS-Feedburner-All-Partial", + "excerpt": "Leslie Jones is back to communicating with her adoring public on Twitter after cowardly hacker-trolls drove her away, probably to compensate for their own failings. It all started with a mic drop ...", + "is_article": "1", + "is_index": "0", + "has_video": "0", + "has_image": "1", + "word_count": "200", + "tags": { + "ifttt": { + "item_id": "1402935436", + "tag": "ifttt" + }, + "mashable": { + "item_id": "1402935436", + "tag": "mashable" + } + }, + "authors": { + "2484273": { + "item_id": "1402935436", + "author_id": "2484273", + "name": "Adam Rosenberg", + "url": "http://mashable.com/author/adam-rosenberg/" + } + }, + "image": { + "item_id": "1402935436", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0" + }, + "images": { + "1": { + "item_id": "1402935436", + "image_id": "1", + "src": "http://i.amz.mshcdn.com/i-V5cS6_sDqFABaVR0hVSBJqG_w=/950x534/https%3A%2F%2Fblueprint-api-production.s3.amazonaws.com%2Fuploads%2Fcard%2Fimage%2F199899%2Fleslie_jones_war_dogs.jpg", + "width": "0", + "height": "0", + "credit": "Image: Steve Eichner/NameFace/Sipa USA", + "caption": "" + } + }, + "userId": 1 +} +JSON; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(1) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn($entry); + + $consumer = new AMQPEntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } + + public function testMessageWithBadUser() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + $entry = new Entry($user); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn(null); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $consumer = new AMQPEntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } + + public function testMessageWithEntryProcessed() + { + $em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $em + ->expects($this->never()) + ->method('flush'); + + $em + ->expects($this->never()) + ->method('clear'); + + $body = '{ "userId": 123 }'; + + $user = new User(); + + $userRepository = $this->getMockBuilder('Wallabag\UserBundle\Repository\UserRepository') + ->disableOriginalConstructor() + ->getMock(); + + $userRepository + ->expects($this->once()) + ->method('find') + // userId from the body json above + ->with(123) + ->willReturn($user); + + $import = $this->getMockBuilder('Wallabag\ImportBundle\Import\AbstractImport') + ->disableOriginalConstructor() + ->getMock(); + + $import + ->expects($this->once()) + ->method('setUser') + ->with($user); + + $import + ->expects($this->once()) + ->method('parseEntry') + ->with(json_decode($body, true)) + ->willReturn(null); + + $consumer = new AMQPEntryConsumer( + $em, + $userRepository, + $import + ); + + $message = new AMQPMessage($body); + + $consumer->execute($message); + } +} -- cgit v1.2.3 From 4fc998245c56ad95c1e753ab52b0c702d4a8a59d Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 20:08:07 +0200 Subject: Only display message in queue for admin Instead of for EVERYONE --- src/Wallabag/ImportBundle/Controller/ImportController.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index d670746c..ea4f7d7b 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -19,12 +19,20 @@ class ImportController extends Controller /** * Display how many messages are queue (both in Redis and RabbitMQ). + * Only for admins. */ public function checkQueueAction() { $nbRedisMessages = null; $nbRabbitMessages = null; + if (!$this->get('security.authorization_checker')->isGranted('ROLE_SUPER_ADMIN')) { + return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [ + 'nbRedisMessages' => $nbRedisMessages, + 'nbRabbitMessages' => $nbRabbitMessages, + ]); + } + if ($this->get('craue_config')->get('import_with_rabbitmq')) { $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket') + $this->getTotalMessageInRabbitQueue('readability') -- cgit v1.2.3 From ebe0787e093f4f2934430033015d6ebad1c64dca Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 22:22:25 +0200 Subject: Moved Pocket token to user config --- app/DoctrineMigrations/Version20160916201049.php | 44 ++++++++++++++++++++++ .../translations/CraueConfigBundle.da.yml | 1 - .../translations/CraueConfigBundle.de.yml | 1 - .../translations/CraueConfigBundle.en.yml | 1 - .../translations/CraueConfigBundle.es.yml | 1 - .../translations/CraueConfigBundle.fa.yml | 1 - .../translations/CraueConfigBundle.fr.yml | 1 - .../translations/CraueConfigBundle.it.yml | 1 - .../translations/CraueConfigBundle.oc.yml | 1 - .../translations/CraueConfigBundle.pl.yml | 1 - .../translations/CraueConfigBundle.ro.yml | 1 - .../translations/CraueConfigBundle.tr.yml | 31 +++++++++++++++ src/Wallabag/CoreBundle/Command/InstallCommand.php | 5 --- .../CoreBundle/DataFixtures/ORM/LoadConfigData.php | 3 ++ .../DataFixtures/ORM/LoadSettingData.php | 5 --- src/Wallabag/CoreBundle/Entity/Config.php | 31 +++++++++++++++ src/Wallabag/CoreBundle/Form/Type/ConfigType.php | 3 ++ .../Resources/translations/messages.da.yml | 1 + .../Resources/translations/messages.de.yml | 1 + .../Resources/translations/messages.en.yml | 1 + .../Resources/translations/messages.es.yml | 1 + .../Resources/translations/messages.fa.yml | 1 + .../Resources/translations/messages.fr.yml | 1 + .../Resources/translations/messages.it.yml | 1 + .../Resources/translations/messages.oc.yml | 1 + .../Resources/translations/messages.pl.yml | 1 + .../Resources/translations/messages.ro.yml | 1 + .../Resources/translations/messages.tr.yml | 1 + .../views/themes/baggy/Config/index.html.twig | 12 ++++++ .../views/themes/material/Config/index.html.twig | 12 ++++++ .../ImportBundle/Controller/PocketController.php | 2 +- src/Wallabag/ImportBundle/Import/PocketImport.php | 10 ++--- .../Resources/views/Pocket/index.html.twig | 8 +--- .../ImportBundle/Import/PocketImportTest.php | 18 ++++----- 34 files changed, 161 insertions(+), 44 deletions(-) create mode 100644 app/DoctrineMigrations/Version20160916201049.php create mode 100644 app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml diff --git a/app/DoctrineMigrations/Version20160916201049.php b/app/DoctrineMigrations/Version20160916201049.php new file mode 100644 index 00000000..ac8e6966 --- /dev/null +++ b/app/DoctrineMigrations/Version20160916201049.php @@ -0,0 +1,44 @@ +container = $container; + } + + private function getTable($tableName) + { + return $this->container->getParameter('database_table_prefix') . $tableName; + } + + /** + * @param Schema $schema + */ + public function up(Schema $schema) + { + $this->addSql('ALTER TABLE '.$this->getTable('config').' ADD pocket_consumer_key VARCHAR(255) DEFAULT NULL'); + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.'); + + $this->addSql('ALTER TABLE `'.$this->getTable('config').'` DROP pocket_consumer_key'); + } +} diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml index 905d7b2b..85079330 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml @@ -8,7 +8,6 @@ export_csv: Aktiver eksport til CSV export_json: Aktiver eksport til JSON export_txt: Aktiver eksport til TXT export_xml: Aktiver eksport til XML -pocket_consumer_key: Brugers nøgle til Pocket for at importere materialer (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli-URL, hvis tjenesten er aktiv diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml index eab50ff2..eaba14cd 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml @@ -8,7 +8,6 @@ export_csv: CSV-Export aktivieren export_json: JSON-Export aktivieren export_txt: TXT-Export aktivieren export_xml: XML-Export aktivieren -pocket_consumer_key: Consumer-Key für Pocket, um Inhalte zu importieren (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli-URL, sofern der Service aktiviert ist diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml index 446c9d05..8aaa27e7 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml @@ -8,7 +8,6 @@ export_csv: Enable CSV export export_json: Enable JSON export export_txt: Enable TXT export export_xml: Enable XML export -pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication) import_with_rabbitmq: Enable RabbitMQ to import data asynchronously import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, if the service is enabled diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml index 42f48bf7..bf3a79af 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml @@ -8,7 +8,6 @@ export_csv: Activar exportación a CSV export_json: Activar exportación a JSON export_txt: Activar exportación a TXT export_xml: Activar exportación a XML -pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, si el servicio está activado diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml index 6fc17c98..d86c4343 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml @@ -8,7 +8,6 @@ export_csv: فعال‌سازی برون‌سپاری به CSV export_json: فعال‌سازی برون‌سپاری به JSON export_txt: فعال‌سازی برون‌سپاری به TXT export_xml: فعال‌سازی برون‌سپاری به XML -pocket_consumer_key: کلید کاربری Pocket برای درون‌ریزی مطالب (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: نشانی Shaarli، اگر فعال بود diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml index 08351b77..5e1ecf26 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml @@ -8,7 +8,6 @@ export_csv: Activer l'export CSV export_json: Activer l'export JSON export_txt: Activer l'export TXT export_xml: Activer l'export XML -pocket_consumer_key: Clé d'authentification Pocket pour importer les données (https://getpocket.com/developer/docs/authentication) import_with_rabbitmq: Activer RabbitMQ pour gérer les imports de façon asynchrone import_with_redis: Activer Redis pour gérer les imports de façon asynchrone shaarli_url: URL de Shaarli, si le service Shaarli est activé diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml index 0c8a906d..9d820e4b 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml @@ -8,7 +8,6 @@ export_csv: Abilita esportazione CSV export_json: Abilita esportazione JSON export_txt: Abilita esportazione TXT export_xml: Abilita esportazione XML -pocket_consumer_key: Consumer key per Pocket per importare i contenuti (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, se il servizio è abilitato diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml index f578095e..49dc7732 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml @@ -8,7 +8,6 @@ export_csv: Activar l'expòrt CSV export_json: Activar l'expòrt JSON export_txt: Activar l'expòrt TXT export_xml: Activar l'expòrt XML -pocket_consumer_key: Clau d'autentificacion Pocket per importar las donadas (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: URL de Shaarli, se lo servici Shaarli es activat diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml index 351ee2a8..2365f2b3 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml @@ -8,7 +8,6 @@ export_csv: Włącz eksport do CSV export_json: Włącz eksport do JSON export_txt: Włącz eksport do TXT export_xml: Włącz eksport do XML -pocket_consumer_key: Klucz klienta Pocket do importu zawartości (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: Adress URL Shaarli, jeżeli usługa jest włączona diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml index b9b41ee7..20d664f7 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml @@ -8,7 +8,6 @@ export_csv: Permite exportare CSV export_json: Permite exportare JSON export_txt: Permite exportare TXT export_xml: Permite exportare XML -pocket_consumer_key: Cheie consumator pentru importarea contentului din Pocket (https://getpocket.com/developer/docs/authentication) # import_with_rabbitmq: Enable RabbitMQ to import data asynchronously # import_with_redis: Enable Redis to import data asynchronously shaarli_url: Shaarli URL, dacă serviciul este permis diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml new file mode 100644 index 00000000..4b4d3edb --- /dev/null +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml @@ -0,0 +1,31 @@ +# download_pictures: Download pictures on your server +# carrot: Enable share to Carrot +# diaspora_url: Diaspora URL, if the service is enabled +# export_epub: Enable ePub export +# export_mobi: Enable .mobi export +# export_pdf: Enable PDF export +# export_csv: Enable CSV export +# export_json: Enable JSON export +# export_txt: Enable TXT export +# export_xml: Enable XML export +# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously +# import_with_redis: Enable Redis to import data asynchronously +# shaarli_url: Shaarli URL, if the service is enabled +# share_diaspora: Enable share to Diaspora +# share_mail: Enable share by email +# share_shaarli: Enable share to Shaarli +# share_twitter: Enable share to Twitter +# show_printlink: Display a link to print content +# wallabag_support_url: Support URL for wallabag +# wallabag_url: URL of *your* wallabag instance +# entry: "article" +# export: "export" +# import: "import" +# misc: "misc" +# modify_settings: "apply" +# piwik_host: Host of your website in Piwik +# piwik_site_id: ID of your website in Piwik +# piwik_enabled: Enable Piwik +# demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" +# demo_mode_username: "Demo user" +# share_public: Allow public url for entries diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 4b1ec02a..bd7b55f9 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -316,11 +316,6 @@ class InstallCommand extends ContainerAwareCommand 'value' => '1', 'section' => 'export', ], - [ - 'name' => 'pocket_consumer_key', - 'value' => null, - 'section' => 'import', - ], [ 'name' => 'import_with_redis', 'value' => '0', diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php index 03be9667..921c739f 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadConfigData.php @@ -20,6 +20,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface $adminConfig->setItemsPerPage(30); $adminConfig->setReadingSpeed(1); $adminConfig->setLanguage('en'); + $adminConfig->setPocketConsumerKey('xxxxx'); $manager->persist($adminConfig); @@ -30,6 +31,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface $bobConfig->setItemsPerPage(10); $bobConfig->setReadingSpeed(1); $bobConfig->setLanguage('fr'); + $bobConfig->setPocketConsumerKey(null); $manager->persist($bobConfig); @@ -40,6 +42,7 @@ class LoadConfigData extends AbstractFixture implements OrderedFixtureInterface $emptyConfig->setItemsPerPage(10); $emptyConfig->setReadingSpeed(1); $emptyConfig->setLanguage('en'); + $emptyConfig->setPocketConsumerKey(null); $manager->persist($emptyConfig); diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index 3ed6bafe..9425f961 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -90,11 +90,6 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface 'value' => '1', 'section' => 'export', ], - [ - 'name' => 'pocket_consumer_key', - 'value' => null, - 'section' => 'import', - ], [ 'name' => 'import_with_redis', 'value' => '0', diff --git a/src/Wallabag/CoreBundle/Entity/Config.php b/src/Wallabag/CoreBundle/Entity/Config.php index a25656d3..d0f0e3f3 100644 --- a/src/Wallabag/CoreBundle/Entity/Config.php +++ b/src/Wallabag/CoreBundle/Entity/Config.php @@ -80,6 +80,13 @@ class Config */ private $readingSpeed; + /** + * @var string + * + * @ORM\Column(name="pocket_consumer_key", type="string", nullable=true) + */ + private $pocketConsumerKey; + /** * @ORM\OneToOne(targetEntity="Wallabag\UserBundle\Entity\User", inversedBy="config") */ @@ -278,6 +285,30 @@ class Config return $this->readingSpeed; } + /** + * Set pocketConsumerKey. + * + * @param string $pocketConsumerKey + * + * @return Config + */ + public function setPocketConsumerKey($pocketConsumerKey) + { + $this->pocketConsumerKey = $pocketConsumerKey; + + return $this; + } + + /** + * Get pocketConsumerKey. + * + * @return string + */ + public function getPocketConsumerKey() + { + return $this->pocketConsumerKey; + } + /** * @param TaggingRule $rule * diff --git a/src/Wallabag/CoreBundle/Form/Type/ConfigType.php b/src/Wallabag/CoreBundle/Form/Type/ConfigType.php index 7d25cc80..0bac2874 100644 --- a/src/Wallabag/CoreBundle/Form/Type/ConfigType.php +++ b/src/Wallabag/CoreBundle/Form/Type/ConfigType.php @@ -52,6 +52,9 @@ class ConfigType extends AbstractType 'choices' => array_flip($this->languages), 'label' => 'config.form_settings.language_label', ]) + ->add('pocket_consumer_key', null, [ + 'label' => 'config.form_settings.pocket_consumer_key_label', + ]) ->add('save', SubmitType::class, [ 'label' => 'config.form.save', ]) diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index 4e5ed9ae..0a7c6e8c 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -68,6 +68,7 @@ config: # 200_word: 'I read ~200 words per minute' # 300_word: 'I read ~300 words per minute' # 400_word: 'I read ~400 words per minute' + pocket_consumer_key_label: Brugers nøgle til Pocket for at importere materialer form_rss: description: 'RSS-feeds fra wallabag gør det muligt at læse de artikler, der gemmes i wallabag, med din RSS-læser. Det kræver, at du genererer et token først.' token_label: 'RSS-Token' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index 107187bf..a400686e 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -68,6 +68,7 @@ config: 200_word: 'Ich lese ~200 Wörter pro Minute' 300_word: 'Ich lese ~300 Wörter pro Minute' 400_word: 'Ich lese ~400 Wörter pro Minute' + pocket_consumer_key_label: Consumer-Key für Pocket, um Inhalte zu importieren form_rss: description: 'Die RSS-Feeds von wallabag erlauben es dir, deine gespeicherten Artikel mit deinem bevorzugten RSS-Reader zu lesen. Vorher musst du jedoch einen Token erstellen.' token_label: 'RSS-token' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index ac60295f..2d097caf 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -68,6 +68,7 @@ config: 200_word: 'I read ~200 words per minute' 300_word: 'I read ~300 words per minute' 400_word: 'I read ~400 words per minute' + pocket_consumer_key_label: Consumer key for Pocket to import contents form_rss: description: 'RSS feeds provided by wallabag allow you to read your saved articles with your favourite RSS reader. You need to generate a token first.' token_label: 'RSS token' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index 1082e6de..1dbff022 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -68,6 +68,7 @@ config: 200_word: 'Leo ~200 palabras por minuto' 300_word: 'Leo ~300 palabras por minuto' 400_word: 'Leo ~400 palabras por minuto' + # pocket_consumer_key_label: Consumer key for Pocket to import contents form_rss: description: 'Los feeds RSS de wallabag permiten leer los artículos guardados con su lector RSS favorito. Necesita generar un token primero' token_label: 'RSS token' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 78317a9c..ad13c940 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -68,6 +68,7 @@ config: 200_word: 'من تقریباً ۲۰۰ واژه را در دقیقه می‌خوانم' 300_word: 'من تقریباً ۳۰۰ واژه را در دقیقه می‌خوانم' 400_word: 'من تقریباً ۴۰۰ واژه را در دقیقه می‌خوانم' + pocket_consumer_key_label: کلید کاربری Pocket برای درون‌ریزی مطالب form_rss: description: 'با خوراک آر-اس-اس که wallabag در اختیارتان می‌گذارد، می‌توانید مقاله‌های ذخیره‌شده را در نرم‌افزار آر-اس-اس دلخواه خود بخوانید. برای این کار نخست باید یک کد بسازید.' token_label: 'کد آر-اس-اس' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index 50515d80..35e5c9d0 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -68,6 +68,7 @@ config: 200_word: 'Je lis environ 200 mots par minute' 300_word: 'Je lis environ 300 mots par minute' 400_word: 'Je lis environ 400 mots par minute' + pocket_consumer_key_label: Clé d'authentification Pocket pour importer les données form_rss: description: "Les flux RSS fournis par wallabag vous permettent de lire vos articles sauvegardés dans votre lecteur de flux préféré. Pour pouvoir les utiliser, vous devez d'abord créer un jeton." token_label: 'Jeton RSS' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 5badeffc..5bc896c3 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -68,6 +68,7 @@ config: 200_word: 'Leggo ~200 parole al minuto' 300_word: 'Leggo ~300 parole al minuto' 400_word: 'Leggo ~400 parole al minuto' + pocket_consumer_key_label: Consumer key per Pocket per importare i contenuti form_rss: description: 'I feed RSS generati da wallabag ti permettono di leggere i tuoi contenuti salvati con il tuo lettore di RSS preferito. Prima, devi generare un token.' token_label: 'RSS token' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index 2383886c..b0194c59 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -68,6 +68,7 @@ config: 200_word: "Legissi a l'entorn de 200 mots per minuta" 300_word: "Legissi a l'entorn de 300 mots per minuta" 400_word: "Legissi a l'entorn de 400 mots per minuta" + pocket_consumer_key_label: Clau d'autentificacion Pocket per importar las donadas form_rss: description: "Los fluxes RSS fornits per wallabag vos permeton de legir vòstres articles salvagardats dins vòstre lector de fluxes preferit. Per los poder emplegar, vos cal, d'en primièr crear un geton." token_label: 'Geton RSS' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index 40cd2b71..6412ad16 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -68,6 +68,7 @@ config: 200_word: 'Czytam ~200 słów na minutę' 300_word: 'Czytam ~300 słów na minutę' 400_word: 'Czytam ~400 słów na minutę' + pocket_consumer_key_label: Klucz klienta Pocket do importu zawartości form_rss: description: 'Kanały RSS prowadzone przez wallabag pozwalają Ci na czytanie twoich zapisanych artykułów w twoium ulubionym czytniku RSS. Musisz najpierw wynegenerować tokena.‌' token_label: 'Token RSS' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index b18eab9e..11b744c7 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -68,6 +68,7 @@ config: # 200_word: 'I read ~200 words per minute' # 300_word: 'I read ~300 words per minute' # 400_word: 'I read ~400 words per minute' + pocket_consumer_key_label: Cheie consumator pentru importarea contentului din Pocket form_rss: description: 'Feed-urile RSS oferite de wallabag îți permit să-ți citești articolele salvate în reader-ul tău preferat RSS.' token_label: 'RSS-Token' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 82fa93af..d6aaacfe 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -68,6 +68,7 @@ config: # 200_word: 'I read ~200 words per minute' # 300_word: 'I read ~300 words per minute' # 400_word: 'I read ~400 words per minute' + # pocket_consumer_key_label: Consumer key for Pocket to import contents form_rss: description: 'wallabag RSS akışı kaydetmiş olduğunuz makalelerini favori RSS okuyucunuzda görüntülemenizi sağlar. Bunu yapabilmek için öncelikle belirteç (token) oluşturmalısınız.' token_label: 'RSS belirteci (token)' diff --git a/src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig b/src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig index f89265d4..6446cf2c 100644 --- a/src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig +++ b/src/Wallabag/CoreBundle/Resources/views/themes/baggy/Config/index.html.twig @@ -44,6 +44,18 @@
+
+
+ {{ form_label(form.config.pocket_consumer_key) }} + {{ form_errors(form.config.pocket_consumer_key) }} + {{ form_widget(form.config.pocket_consumer_key) }} +

+ » + https://getpocket.com/developer/docs/authentication +

+
+
+ {{ form_rest(form.config) }} diff --git a/src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig b/src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig index bf390e89..5330c353 100644 --- a/src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig +++ b/src/Wallabag/CoreBundle/Resources/views/themes/material/Config/index.html.twig @@ -62,6 +62,18 @@
+
+
+ {{ form_label(form.config.pocket_consumer_key) }} + {{ form_errors(form.config.pocket_consumer_key) }} + {{ form_widget(form.config.pocket_consumer_key) }} +

+ » + https://getpocket.com/developer/docs/authentication +

+
+
+ {{ form_widget(form.config.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} {{ form_rest(form.config) }} diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 1f92c182..56be5cbf 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php @@ -44,7 +44,7 @@ class PocketController extends Controller return $this->render('WallabagImportBundle:Pocket:index.html.twig', [ 'import' => $this->getPocketImportService(), - 'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true, + 'has_consumer_key' => '' === trim($this->getUser()->getConfig()->getPocketConsumerKey()) ? false : true, 'form' => $form->createView(), ]); } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index cc6faf1f..40603c90 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -13,16 +13,14 @@ use Craue\ConfigBundle\Util\Config; class PocketImport extends AbstractImport { private $client; - private $consumerKey; private $accessToken; const NB_ELEMENTS = 5000; - public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) + public function __construct(EntityManager $em, ContentProxy $contentProxy) { $this->em = $em; $this->contentProxy = $contentProxy; - $this->consumerKey = $craueConfig->get('pocket_consumer_key'); $this->logger = new NullLogger(); } @@ -72,7 +70,7 @@ class PocketImport extends AbstractImport $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'redirect_uri' => $redirectUri, ]), ] @@ -102,7 +100,7 @@ class PocketImport extends AbstractImport $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'code' => $code, ]), ] @@ -131,7 +129,7 @@ class PocketImport extends AbstractImport $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', [ 'body' => json_encode([ - 'consumer_key' => $this->consumerKey, + 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), 'access_token' => $this->accessToken, 'detailType' => 'complete', 'state' => 'all', diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig index aa5941b7..6195fa07 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig @@ -12,11 +12,7 @@
{{ 'import.pocket.config_missing.description'|trans }} - {% if is_granted('ROLE_SUPER_ADMIN') %} - {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '', '%keyurle%':''})|raw }} - {% else %} - {{ 'import.pocket.config_missing.user_message'|trans }} - {% endif %} + {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '', '%keyurle%':''})|raw }}
{% endif %} @@ -31,7 +27,7 @@ {{ form_label(form.mark_as_read) }}
- diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index a3f68892..48fbbfb6 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -4,6 +4,7 @@ namespace Tests\Wallabag\ImportBundle\Import; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\CoreBundle\Entity\Config; use Wallabag\ImportBundle\Import\PocketImport; use GuzzleHttp\Client; use GuzzleHttp\Subscriber\Mock; @@ -27,6 +28,11 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase { $this->user = new User(); + $config = new Config($this->user); + $config->setPocketConsumerKey('xxx'); + + $this->user->setConfig($config); + $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') ->disableOriginalConstructor() ->getMock(); @@ -35,19 +41,9 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $config = $this->getMockBuilder('Craue\ConfigBundle\Util\Config') - ->disableOriginalConstructor() - ->getMock(); - - $config->expects($this->any()) - ->method('get') - ->with('pocket_consumer_key') - ->willReturn($consumerKey); - $pocket = new PocketImport( $this->em, - $this->contentProxy, - $config + $this->contentProxy ); $pocket->setUser($this->user); -- cgit v1.2.3 From fbb319f064e6336a3b44bda12cdc51c93c51f379 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 16 Sep 2016 22:58:33 +0200 Subject: Missing some migrations and CS --- app/DoctrineMigrations/Version20160916201049.php | 2 ++ src/Wallabag/ImportBundle/Import/PocketImport.php | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/DoctrineMigrations/Version20160916201049.php b/app/DoctrineMigrations/Version20160916201049.php index ac8e6966..202901e6 100644 --- a/app/DoctrineMigrations/Version20160916201049.php +++ b/app/DoctrineMigrations/Version20160916201049.php @@ -30,6 +30,7 @@ class Version20160916201049 extends AbstractMigration implements ContainerAwareI public function up(Schema $schema) { $this->addSql('ALTER TABLE '.$this->getTable('config').' ADD pocket_consumer_key VARCHAR(255) DEFAULT NULL'); + $this->addSql("DELETE FROM `".$this->getTable('craue_config_setting')."` WHERE `name` = 'pocket_consumer_key';"); } /** @@ -40,5 +41,6 @@ class Version20160916201049 extends AbstractMigration implements ContainerAwareI $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.'); $this->addSql('ALTER TABLE `'.$this->getTable('config').'` DROP pocket_consumer_key'); + $this->addSql("INSERT INTO `".$this->getTable('craue_config_setting')."` (`name`, `value`, `section`) VALUES ('pocket_consumer_key', NULL, 'import')"); } } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 40603c90..1bf22d68 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -8,7 +8,6 @@ use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Helper\ContentProxy; -use Craue\ConfigBundle\Util\Config; class PocketImport extends AbstractImport { -- cgit v1.2.3 From 59b97fae996d8307b9d957d210d46200f6d206bf Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 17 Sep 2016 07:40:56 +0200 Subject: Avoid losing entry when fetching fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of just say “Failed to save entry” we’ll save the entry at all cost and try to fetch content. If fetching content failed, the entry will still be saved at least, but without content. --- .../CoreBundle/Controller/EntryController.php | 50 +++++++++++++--------- .../Resources/translations/messages.da.yml | 4 +- .../Resources/translations/messages.de.yml | 4 +- .../Resources/translations/messages.en.yml | 4 +- .../Resources/translations/messages.es.yml | 4 +- .../Resources/translations/messages.fa.yml | 4 +- .../Resources/translations/messages.fr.yml | 4 +- .../Resources/translations/messages.it.yml | 4 +- .../Resources/translations/messages.oc.yml | 4 +- .../Resources/translations/messages.pl.yml | 4 +- .../Resources/translations/messages.ro.yml | 4 +- .../Resources/translations/messages.tr.yml | 4 +- .../ImportBundle/Import/AbstractImport.php | 6 +-- src/Wallabag/ImportBundle/Import/PocketImport.php | 19 +++----- .../ImportBundle/Import/ReadabilityImport.php | 16 +++---- .../ImportBundle/Import/WallabagImport.php | 16 +++---- .../ImportBundle/Import/PocketImportTest.php | 4 +- .../ImportBundle/Import/WallabagV2ImportTest.php | 2 +- 18 files changed, 73 insertions(+), 84 deletions(-) diff --git a/src/Wallabag/CoreBundle/Controller/EntryController.php b/src/Wallabag/CoreBundle/Controller/EntryController.php index 624576b5..40111af0 100644 --- a/src/Wallabag/CoreBundle/Controller/EntryController.php +++ b/src/Wallabag/CoreBundle/Controller/EntryController.php @@ -17,26 +17,35 @@ use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache; class EntryController extends Controller { /** - * @param Entry $entry + * Fetch content and update entry. + * In case it fails, entry will return to avod loosing the data. + * + * @param Entry $entry + * @param string $prefixMessage Should be the translation key: entry_saved or entry_reloaded + * + * @return Entry */ - private function updateEntry(Entry $entry) + private function updateEntry(Entry $entry, $prefixMessage = 'entry_saved') { + // put default title in case of fetching content failed + $entry->setTitle('No title found'); + + $message = 'flashes.entry.notice.'.$prefixMessage; + try { $entry = $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl()); - - $em = $this->getDoctrine()->getManager(); - $em->persist($entry); - $em->flush(); } catch (\Exception $e) { $this->get('logger')->error('Error while saving an entry', [ 'exception' => $e, 'entry' => $entry, ]); - return false; + $message = 'flashes.entry.notice.'.$prefixMessage.'_failed'; } - return true; + $this->get('session')->getFlashBag()->add('notice', $message); + + return $entry; } /** @@ -66,12 +75,11 @@ class EntryController extends Controller return $this->redirect($this->generateUrl('view', ['id' => $existingEntry->getId()])); } - $message = 'flashes.entry.notice.entry_saved'; - if (false === $this->updateEntry($entry)) { - $message = 'flashes.entry.notice.entry_saved_failed'; - } + $this->updateEntry($entry); - $this->get('session')->getFlashBag()->add('notice', $message); + $em = $this->getDoctrine()->getManager(); + $em->persist($entry); + $em->flush(); return $this->redirect($this->generateUrl('homepage')); } @@ -95,6 +103,10 @@ class EntryController extends Controller if (false === $this->checkIfEntryAlreadyExists($entry)) { $this->updateEntry($entry); + + $em = $this->getDoctrine()->getManager(); + $em->persist($entry); + $em->flush(); } return $this->redirect($this->generateUrl('homepage')); @@ -316,15 +328,11 @@ class EntryController extends Controller { $this->checkUserAction($entry); - $message = 'flashes.entry.notice.entry_reloaded'; - if (false === $this->updateEntry($entry)) { - $message = 'flashes.entry.notice.entry_reload_failed'; - } + $this->updateEntry($entry, 'entry_reloaded'); - $this->get('session')->getFlashBag()->add( - 'notice', - $message - ); + $em = $this->getDoctrine()->getManager(); + $em->persist($entry); + $em->flush(); return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()])); } diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index 0a7c6e8c..9f051edb 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -414,10 +414,10 @@ flashes: notice: # entry_already_saved: 'Entry already saved on %date%' # entry_saved: 'Entry saved' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' # entry_updated: 'Entry updated' # entry_reloaded: 'Entry reloaded' - # entry_reload_failed: 'Failed to reload entry' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Artikel arkiveret' entry_unarchived: 'Artikel ikke længere arkiveret' entry_starred: 'Artikel markeret som favorit' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index a400686e..cbfacd55 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -414,10 +414,10 @@ flashes: notice: entry_already_saved: 'Eintrag bereits am %date% gespeichert' entry_saved: 'Eintrag gespeichert' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'Eintrag aktualisiert' entry_reloaded: 'Eintrag neugeladen' - entry_reload_failed: 'Neuladen des Eintrags fehlgeschlagen' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Artikel archiviert' entry_unarchived: 'Artikel dearchiviert' entry_starred: 'Artikel favorisiert' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 2d097caf..21e2405c 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -416,10 +416,10 @@ flashes: notice: entry_already_saved: 'Entry already saved on %date%' entry_saved: 'Entry saved' - entry_saved_failed: 'Failed to save entry' + entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'Entry updated' entry_reloaded: 'Entry reloaded' - entry_reload_failed: 'Failed to reload entry' + entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Entry archived' entry_unarchived: 'Entry unarchived' entry_starred: 'Entry starred' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index 1dbff022..43f376d4 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -414,10 +414,10 @@ flashes: notice: entry_already_saved: 'Entrada ya guardada por %fecha%' entry_saved: 'Entrada guardada' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'Entrada actualizada' entry_reloaded: 'Entrada recargada' - entry_reload_failed: 'Entrada recargada reprobada' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Artículo archivado' entry_unarchived: 'Artículo desarchivado' entry_starred: 'Artículo guardado en los favoritos' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index ad13c940..56418ef9 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -414,10 +414,10 @@ flashes: notice: entry_already_saved: 'این مقاله در تاریخ %date% ذخیره شده بود' entry_saved: 'مقاله ذخیره شد' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'مقاله به‌روز شد' entry_reloaded: 'مقاله به‌روز شد' - entry_reload_failed: 'به‌روزرسانی مقاله شکست خورد' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'مقاله بایگانی شد' entry_unarchived: 'مقاله از بایگانی درآمد' entry_starred: 'مقاله برگزیده شد' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index 35e5c9d0..bde21866 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -416,10 +416,10 @@ flashes: notice: entry_already_saved: 'Article déjà sauvergardé le %date%' entry_saved: 'Article enregistré' - entry_saved_failed: "L'enregistrement a échoué" + entry_saved_failed: 'Article enregistré mais impossible de récupérer le contenu' entry_updated: 'Article mis à jour' entry_reloaded: 'Article rechargé' - entry_reload_failed: "Le rechargement de l'article a échoué" + entry_reload_failed: "Article mis à jour mais impossible de récupérer le contenu" entry_archived: 'Article marqué comme lu' entry_unarchived: 'Article marqué comme non lu' entry_starred: 'Article ajouté dans les favoris' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 5bc896c3..26bb31ba 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -413,10 +413,10 @@ flashes: notice: entry_already_saved: 'Contenuto già salvato in data %date%' entry_saved: 'Contenuto salvato' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'Contenuto aggiornato' entry_reloaded: 'Contenuto ricaricato' - entry_reload_failed: 'Errore nel ricaricamento del contenuto' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Contenuto archiviato' entry_unarchived: 'Contenuto dis-archiviato' entry_starred: 'Contenuto segnato come preferito' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index b0194c59..2c4df867 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -414,10 +414,10 @@ flashes: notice: entry_already_saved: 'Article ja salvargardat lo %date%' entry_saved: 'Article enregistrat' - entry_saved_failed: "Fracàs de l'enregistrament de l'entrada" + # entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'Article mes a jorn' entry_reloaded: 'Article recargat' - entry_reload_failed: "Fracàs de l'actualizacion de l'article" + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Article marcat coma legit' entry_unarchived: 'Article marcat coma pas legit' entry_starred: 'Article apondut dins los favorits' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index 6412ad16..fb821966 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -414,10 +414,10 @@ flashes: notice: entry_already_saved: 'Wpis już został dodany %date%' entry_saved: 'Wpis zapisany' - entry_saved_failed: 'Zapis artykułu się nie powiódł' + # entry_saved_failed: 'Entry saved but fetching content failed' entry_updated: 'Wpis zaktualizowany' entry_reloaded: 'Wpis ponownie załadowany' - entry_reload_failed: 'Błąd ponownego załadowania' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Wpis dodany do archiwum' entry_unarchived: 'Wpis usunięty z archiwum' entry_starred: 'Wpis oznaczony gwiazdką' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 11b744c7..3d22e29d 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -414,10 +414,10 @@ flashes: notice: # entry_already_saved: 'Entry already saved on %date%' # entry_saved: 'Entry saved' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' # entry_updated: 'Entry updated' # entry_reloaded: 'Entry reloaded' - # entry_reload_failed: 'Failed to reload entry' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Articol arhivat' entry_unarchived: 'Articol dezarhivat' entry_starred: 'Articol adăugat la favorite' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index d6aaacfe..5099b002 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -414,10 +414,10 @@ flashes: notice: entry_already_saved: 'Entry already saved on %date%' entry_saved: 'Makale kaydedildi' - # entry_saved_failed: 'Failed to save entry' + # entry_saved_failed: 'Entry saved but fetching content failed' # entry_updated: 'Entry updated' entry_reloaded: 'Makale içeriği yenilendi' - # entry_reload_failed: 'Failed to reload entry' + # entry_reload_failed: 'Entry reloaded but fetching content failed' entry_archived: 'Makale arşivlendi' entry_unarchived: 'Makale arşivden çıkartıldı' entry_starred: 'Makale favorilere eklendi' diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 2af0e69b..a1a14576 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -79,20 +79,20 @@ abstract class AbstractImport implements ImportInterface /** * Fetch content from the ContentProxy (using graby). - * If it fails return false instead of the updated entry. + * If it fails return the given entry to be saved in all case (to avoid user to loose the content). * * @param Entry $entry Entry to update * @param string $url Url to grab content for * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url * - * @return Entry|false + * @return Entry */ protected function fetchContent(Entry $entry, $url, array $content = []) { try { return $this->contentProxy->updateEntry($entry, $url, $content); } catch (\Exception $e) { - return false; + return $entry; } } diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 1bf22d68..e00eb44b 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -199,24 +199,16 @@ class PocketImport extends AbstractImport } $entry = new Entry($this->user); - $entry = $this->fetchContent($entry, $url); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry->setUrl($url); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $url); // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted - if ($importedEntry['status'] == 1 || $this->markAsRead) { - $entry->setArchived(true); - } + $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead); // 0 or 1 - 1 If the item is starred - if ($importedEntry['favorite'] == 1) { - $entry->setStarred(true); - } + $entry->setStarred($importedEntry['favorite'] == 1); $title = 'Untitled'; if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') { @@ -226,7 +218,6 @@ class PocketImport extends AbstractImport } $entry->setTitle($title); - $entry->setUrl($url); // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php index b852f8f0..fa2b7053 100644 --- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -103,18 +103,12 @@ class ReadabilityImport extends AbstractImport 'created_at' => $importedEntry['date_added'], ]; - $entry = $this->fetchContent( - new Entry($this->user), - $data['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); $entry->setArchived($data['is_archived']); $entry->setStarred($data['is_starred']); diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index 969a6a04..043bb0a2 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php @@ -101,18 +101,12 @@ abstract class WallabagImport extends AbstractImport $data = $this->prepareEntry($importedEntry); - $entry = $this->fetchContent( - new Entry($this->user), - $importedEntry['url'], - $data - ); - - // jump to next entry in case of problem while getting content - if (false === $entry) { - ++$this->skippedEntries; + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - return; - } + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); if (array_key_exists('tags', $data)) { $this->contentProxy->assignTagsToEntry( diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 48fbbfb6..952521a2 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -566,6 +566,8 @@ JSON; "status": 1, "list": { "229279689": { + "status": "1", + "favorite": "1", "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview" } } @@ -603,6 +605,6 @@ JSON; $res = $pocketImport->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 1, 'imported' => 0, 'queued' => 0], $pocketImport->getSummary()); + $this->assertEquals(['skipped' => 0, 'imported' => 1, 'queued' => 0], $pocketImport->getSummary()); } } diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php index b4017f72..12bd6bdd 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php @@ -256,6 +256,6 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase $res = $wallabagV2Import->import(); $this->assertTrue($res); - $this->assertEquals(['skipped' => 24, 'imported' => 0, 'queued' => 0], $wallabagV2Import->getSummary()); + $this->assertEquals(['skipped' => 22, 'imported' => 2, 'queued' => 0], $wallabagV2Import->getSummary()); } } -- cgit v1.2.3