From ff1a5362f7254d686864ea53994da6c517b3d3e8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 27 Sep 2016 07:57:53 +0200 Subject: Add Instapaper import Also update ImportController with latest import (chrome, firefox & instapaper). --- app/config/config.yml | 2 +- .../ImportBundle/Command/RedisWorkerCommand.php | 2 +- .../ImportBundle/Controller/ImportController.php | 12 +- .../Controller/InstapaperController.php | 77 +++++++ .../ImportBundle/Import/AbstractImport.php | 4 + .../ImportBundle/Import/InstapaperImport.php | 134 ++++++++++++ .../ImportBundle/Resources/config/rabbit.yml | 7 + .../ImportBundle/Resources/config/redis.yml | 20 ++ .../ImportBundle/Resources/config/services.yml | 10 + .../Resources/views/Instapaper/index.html.twig | 45 ++++ .../Controller/ImportControllerTest.php | 2 +- .../ImportBundle/Import/InstapaperImportTest.php | 233 +++++++++++++++++++++ .../ImportBundle/fixtures/instapaper-export.csv | 4 + 13 files changed, 547 insertions(+), 5 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/InstapaperController.php create mode 100644 src/Wallabag/ImportBundle/Import/InstapaperImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig create mode 100644 tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php create mode 100644 tests/Wallabag/ImportBundle/fixtures/instapaper-export.csv diff --git a/app/config/config.yml b/app/config/config.yml index a4584a1b..cfb0d54d 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -55,7 +55,7 @@ wallabag_user: registration_enabled: "%fosuser_registration%" wallabag_import: - allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain'] + allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv'] resource_dir: "%kernel.root_dir%/../web/uploads/import" # Twig Configuration diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php index 5f90e00f..c2c11f11 100644 --- a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php +++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php @@ -17,7 +17,7 @@ class RedisWorkerCommand extends ContainerAwareCommand $this ->setName('wallabag:import:redis-worker') ->setDescription('Launch Redis worker') - ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability') + ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, firefox, chrome or instapaper') ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false) ; } diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 36a2a399..15de75ff 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -38,7 +38,11 @@ class ImportController extends Controller $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket') + $this->getTotalMessageInRabbitQueue('readability') + $this->getTotalMessageInRabbitQueue('wallabag_v1') - + $this->getTotalMessageInRabbitQueue('wallabag_v2'); + + $this->getTotalMessageInRabbitQueue('wallabag_v2') + + $this->getTotalMessageInRabbitQueue('firefox') + + $this->getTotalMessageInRabbitQueue('chrome') + + $this->getTotalMessageInRabbitQueue('instapaper') + ; } catch (\Exception $e) { $rabbitNotInstalled = true; } @@ -49,7 +53,11 @@ class ImportController extends Controller $nbRedisMessages = $redis->llen('wallabag.import.pocket') + $redis->llen('wallabag.import.readability') + $redis->llen('wallabag.import.wallabag_v1') - + $redis->llen('wallabag.import.wallabag_v2'); + + $redis->llen('wallabag.import.wallabag_v2') + + $redis->llen('wallabag.import.firefox') + + $redis->llen('wallabag.import.chrome') + + $redis->llen('wallabag.import.instapaper') + ; } catch (\Exception $e) { $redisNotInstalled = true; } diff --git a/src/Wallabag/ImportBundle/Controller/InstapaperController.php b/src/Wallabag/ImportBundle/Controller/InstapaperController.php new file mode 100644 index 00000000..c3fc8a39 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/InstapaperController.php @@ -0,0 +1,77 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $instapaper = $this->get('wallabag_import.instapaper.import'); + $instapaper->setUser($this->getUser()); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $instapaper->setProducer($this->get('old_sound_rabbit_mq.import_instapaper_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $instapaper->setProducer($this->get('wallabag_import.producer.redis.instapaper')); + } + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = 'instapaper_'.$this->getUser()->getId().'.csv'; + + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $instapaper + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $instapaper->getSummary(); + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render('WallabagImportBundle:Instapaper:index.html.twig', [ + 'form' => $form->createView(), + 'import' => $instapaper, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index a1a14576..764b390a 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -106,6 +106,10 @@ abstract class AbstractImport implements ImportInterface $i = 1; foreach ($entries as $importedEntry) { + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + $entry = $this->parseEntry($importedEntry); if (null === $entry) { diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php new file mode 100644 index 00000000..356acf23 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php @@ -0,0 +1,134 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('InstapaperImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('InstapaperImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $entries = []; + $handle = fopen($this->filepath, 'r'); + while (($data = fgetcsv($handle, 10240)) !== false) { + if ('URL' === $data[0]) { + continue; + } + + $entries[] = [ + 'url' => $data[0], + 'title' => $data[1], + 'status' => $data[3], + 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred', + 'is_starred' => $data[3] === 'Starred', + 'content_type' => '', + 'language' => '', + ]; + } + fclose($handle); + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $entry = new Entry($this->user); + $entry->setUrl($importedEntry['url']); + $entry->setTitle($importedEntry['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry); + + $entry->setArchived($importedEntry['is_archived']); + $entry->setStarred($importedEntry['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index 6ada6302..70b8a0d4 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -14,6 +14,13 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" - "@logger" + wallabag_import.consumer.amqp.instapaper: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.instapaper.import" + - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index c9c2cf26..0a81e1b5 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -20,6 +20,26 @@ services: - "@wallabag_import.readability.import" - "@logger" + # instapaper + wallabag_import.queue.redis.instapaper: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.instapaper" + + wallabag_import.producer.redis.instapaper: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.instapaper" + + wallabag_import.consumer.redis.instapaper: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.instapaper.import" + - "@logger" + # pocket wallabag_import.queue.redis.pocket: class: Simpleue\Queue\RedisQueue diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 990f336d..89adc71b 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -57,6 +57,16 @@ services: tags: - { name: wallabag_import.import, alias: readability } + wallabag_import.instapaper.import: + class: Wallabag\ImportBundle\Import\InstapaperImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: instapaper } + wallabag_import.firefox.import: class: Wallabag\ImportBundle\Import\FirefoxImport arguments: diff --git a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig new file mode 100644 index 00000000..5789361f --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig @@ -0,0 +1,45 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.instapaper.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + +
+
{{ import.description|trans }}
+

{{ 'import.instapaper.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php index b6783a56..0bc40bdd 100644 --- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php @@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase $crawler = $client->request('GET', '/import/'); $this->assertEquals(200, $client->getResponse()->getStatusCode()); - $this->assertEquals(6, $crawler->filter('blockquote')->count()); + $this->assertEquals(7, $crawler->filter('blockquote')->count()); } } diff --git a/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php b/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php new file mode 100644 index 00000000..75900bd7 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php @@ -0,0 +1,233 @@ +user = new User(); + + $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') + ->disableOriginalConstructor() + ->getMock(); + + $import = new InstapaperImport($this->em, $this->contentProxy); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + $import->setLogger($logger); + + if (false === $unsetUser) { + $import->setUser($this->user); + } + + return $import; + } + + public function testInit() + { + $instapaperImport = $this->getInstapaperImport(); + + $this->assertEquals('Instapaper', $instapaperImport->getName()); + $this->assertNotEmpty($instapaperImport->getUrl()); + $this->assertEquals('import.instapaper.description', $instapaperImport->getDescription()); + } + + public function testImport() + { + $instapaperImport = $this->getInstapaperImport(); + $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(3)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(3)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $instapaperImport->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 3, 'queued' => 0], $instapaperImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $instapaperImport = $this->getInstapaperImport(); + $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(3)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->once()) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->once()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return $persistedEntry->isArchived(); + })); + + $res = $instapaperImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + + $this->assertEquals(['skipped' => 2, 'imported' => 1, 'queued' => 0], $instapaperImport->getSummary()); + } + + public function testImportWithRabbit() + { + $instapaperImport = $this->getInstapaperImport(); + $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(3)) + ->method('publish'); + + $instapaperImport->setProducer($producer); + + $res = $instapaperImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 3], $instapaperImport->getSummary()); + } + + public function testImportWithRedis() + { + $instapaperImport = $this->getInstapaperImport(); + $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'instapaper'); + $producer = new Producer($queue); + + $instapaperImport->setProducer($producer); + + $res = $instapaperImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 3], $instapaperImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('instapaper')); + } + + public function testImportBadFile() + { + $instapaperImport = $this->getInstapaperImport(); + $instapaperImport->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx'); + + $res = $instapaperImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('InstapaperImport: unable to read file', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $instapaperImport = $this->getInstapaperImport(true); + $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); + + $res = $instapaperImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('InstapaperImport: user is not defined', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } +} diff --git a/tests/Wallabag/ImportBundle/fixtures/instapaper-export.csv b/tests/Wallabag/ImportBundle/fixtures/instapaper-export.csv new file mode 100644 index 00000000..28a4c8e6 --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/instapaper-export.csv @@ -0,0 +1,4 @@ +URL,Title,Selection,Folder +http://www.liberation.fr/societe/2012/12/06/baumettes-un-tour-en-cellule_865551,Baumettes : un tour en cellule,,Unread +https://redditblog.com/2016/09/20/amp-and-reactredux/,AMP and React+Redux: Why Not?,,Archive +https://medium.com/@the_minh/why-foursquare-swarm-is-still-my-favourite-social-network-e38228493e6c,Why Foursquare / Swarm is still my favourite social network,,Starred -- cgit v1.2.3