Also update ImportController with latest import (chrome, firefox & instapaper).
registration_enabled: "%fosuser_registration%"
wallabag_import:
- allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain']
+ allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv']
resource_dir: "%kernel.root_dir%/../web/uploads/import"
# Twig Configuration
$this
->setName('wallabag:import:redis-worker')
->setDescription('Launch Redis worker')
- ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket or readability')
+ ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, firefox, chrome or instapaper')
->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false)
;
}
$nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket')
+ $this->getTotalMessageInRabbitQueue('readability')
+ $this->getTotalMessageInRabbitQueue('wallabag_v1')
- + $this->getTotalMessageInRabbitQueue('wallabag_v2');
+ + $this->getTotalMessageInRabbitQueue('wallabag_v2')
+ + $this->getTotalMessageInRabbitQueue('firefox')
+ + $this->getTotalMessageInRabbitQueue('chrome')
+ + $this->getTotalMessageInRabbitQueue('instapaper')
+ ;
} catch (\Exception $e) {
$rabbitNotInstalled = true;
}
$nbRedisMessages = $redis->llen('wallabag.import.pocket')
+ $redis->llen('wallabag.import.readability')
+ $redis->llen('wallabag.import.wallabag_v1')
- + $redis->llen('wallabag.import.wallabag_v2');
+ + $redis->llen('wallabag.import.wallabag_v2')
+ + $redis->llen('wallabag.import.firefox')
+ + $redis->llen('wallabag.import.chrome')
+ + $redis->llen('wallabag.import.instapaper')
+ ;
} catch (\Exception $e) {
$redisNotInstalled = true;
}
--- /dev/null
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Symfony\Bundle\FrameworkBundle\Controller\Controller;
+use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
+use Symfony\Component\HttpFoundation\Request;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+
+class InstapaperController extends Controller
+{
+ /**
+ * @Route("/instapaper", name="import_instapaper")
+ */
+ public function indexAction(Request $request)
+ {
+ $form = $this->createForm(UploadImportType::class);
+ $form->handleRequest($request);
+
+ $instapaper = $this->get('wallabag_import.instapaper.import');
+ $instapaper->setUser($this->getUser());
+
+ if ($this->get('craue_config')->get('import_with_rabbitmq')) {
+ $instapaper->setProducer($this->get('old_sound_rabbit_mq.import_instapaper_producer'));
+ } elseif ($this->get('craue_config')->get('import_with_redis')) {
+ $instapaper->setProducer($this->get('wallabag_import.producer.redis.instapaper'));
+ }
+
+ if ($form->isValid()) {
+ $file = $form->get('file')->getData();
+ $markAsRead = $form->get('mark_as_read')->getData();
+ $name = 'instapaper_'.$this->getUser()->getId().'.csv';
+
+ if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+ $res = $instapaper
+ ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
+ ->setMarkAsRead($markAsRead)
+ ->import();
+
+ $message = 'flashes.import.notice.failed';
+
+ if (true === $res) {
+ $summary = $instapaper->getSummary();
+ $message = $this->get('translator')->trans('flashes.import.notice.summary', [
+ '%imported%' => $summary['imported'],
+ '%skipped%' => $summary['skipped'],
+ ]);
+
+ if (0 < $summary['queued']) {
+ $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
+ '%queued%' => $summary['queued'],
+ ]);
+ }
+
+ unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
+ }
+
+ $this->get('session')->getFlashBag()->add(
+ 'notice',
+ $message
+ );
+
+ return $this->redirect($this->generateUrl('homepage'));
+ } else {
+ $this->get('session')->getFlashBag()->add(
+ 'notice',
+ 'flashes.import.notice.failed_on_file'
+ );
+ }
+ }
+
+ return $this->render('WallabagImportBundle:Instapaper:index.html.twig', [
+ 'form' => $form->createView(),
+ 'import' => $instapaper,
+ ]);
+ }
+}
$i = 1;
foreach ($entries as $importedEntry) {
+ if ($this->markAsRead) {
+ $importedEntry = $this->setEntryAsRead($importedEntry);
+ }
+
$entry = $this->parseEntry($importedEntry);
if (null === $entry) {
--- /dev/null
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Wallabag\CoreBundle\Entity\Entry;
+
+class InstapaperImport extends AbstractImport
+{
+ private $filepath;
+
+ /**
+ * {@inheritdoc}
+ */
+ public function getName()
+ {
+ return 'Instapaper';
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public function getUrl()
+ {
+ return 'import_instapaper';
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public function getDescription()
+ {
+ return 'import.instapaper.description';
+ }
+
+ /**
+ * Set file path to the json file.
+ *
+ * @param string $filepath
+ */
+ public function setFilepath($filepath)
+ {
+ $this->filepath = $filepath;
+
+ return $this;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public function import()
+ {
+ if (!$this->user) {
+ $this->logger->error('InstapaperImport: user is not defined');
+
+ return false;
+ }
+
+ if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+ $this->logger->error('InstapaperImport: unable to read file', ['filepath' => $this->filepath]);
+
+ return false;
+ }
+
+ $entries = [];
+ $handle = fopen($this->filepath, 'r');
+ while (($data = fgetcsv($handle, 10240)) !== false) {
+ if ('URL' === $data[0]) {
+ continue;
+ }
+
+ $entries[] = [
+ 'url' => $data[0],
+ 'title' => $data[1],
+ 'status' => $data[3],
+ 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred',
+ 'is_starred' => $data[3] === 'Starred',
+ 'content_type' => '',
+ 'language' => '',
+ ];
+ }
+ fclose($handle);
+
+ if ($this->producer) {
+ $this->parseEntriesForProducer($entries);
+
+ return true;
+ }
+
+ $this->parseEntries($entries);
+
+ return true;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ public function parseEntry(array $importedEntry)
+ {
+ $existingEntry = $this->em
+ ->getRepository('WallabagCoreBundle:Entry')
+ ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
+
+ if (false !== $existingEntry) {
+ ++$this->skippedEntries;
+
+ return;
+ }
+
+ $entry = new Entry($this->user);
+ $entry->setUrl($importedEntry['url']);
+ $entry->setTitle($importedEntry['title']);
+
+ // update entry with content (in case fetching failed, the given entry will be return)
+ $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry);
+
+ $entry->setArchived($importedEntry['is_archived']);
+ $entry->setStarred($importedEntry['is_starred']);
+
+ $this->em->persist($entry);
+ ++$this->importedEntries;
+
+ return $entry;
+ }
+
+ /**
+ * {@inheritdoc}
+ */
+ protected function setEntryAsRead(array $importedEntry)
+ {
+ $importedEntry['is_archived'] = 1;
+
+ return $importedEntry;
+ }
+}
- "@wallabag_user.user_repository"
- "@wallabag_import.readability.import"
- "@logger"
+ wallabag_import.consumer.amqp.instapaper:
+ class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.instapaper.import"
+ - "@logger"
wallabag_import.consumer.amqp.wallabag_v1:
class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
arguments:
- "@wallabag_import.readability.import"
- "@logger"
+ # instapaper
+ wallabag_import.queue.redis.instapaper:
+ class: Simpleue\Queue\RedisQueue
+ arguments:
+ - "@wallabag_core.redis.client"
+ - "wallabag.import.instapaper"
+
+ wallabag_import.producer.redis.instapaper:
+ class: Wallabag\ImportBundle\Redis\Producer
+ arguments:
+ - "@wallabag_import.queue.redis.instapaper"
+
+ wallabag_import.consumer.redis.instapaper:
+ class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_user.user_repository"
+ - "@wallabag_import.instapaper.import"
+ - "@logger"
+
# pocket
wallabag_import.queue.redis.pocket:
class: Simpleue\Queue\RedisQueue
tags:
- { name: wallabag_import.import, alias: readability }
+ wallabag_import.instapaper.import:
+ class: Wallabag\ImportBundle\Import\InstapaperImport
+ arguments:
+ - "@doctrine.orm.entity_manager"
+ - "@wallabag_core.content_proxy"
+ calls:
+ - [ setLogger, [ "@logger" ]]
+ tags:
+ - { name: wallabag_import.import, alias: instapaper }
+
wallabag_import.firefox.import:
class: Wallabag\ImportBundle\Import\FirefoxImport
arguments:
--- /dev/null
+{% extends "WallabagCoreBundle::layout.html.twig" %}
+
+{% block title %}{{ 'import.instapaper.page_title'|trans }}{% endblock %}
+
+{% block content %}
+<div class="row">
+ <div class="col s12">
+ <div class="card-panel settings">
+ {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
+ <div class="row">
+ <blockquote>{{ import.description|trans }}</blockquote>
+ <p>{{ 'import.instapaper.how_to'|trans }}</p>
+
+ <div class="col s12">
+ {{ form_start(form, {'method': 'POST'}) }}
+ {{ form_errors(form) }}
+ <div class="row">
+ <div class="file-field input-field col s12">
+ {{ form_errors(form.file) }}
+ <div class="btn">
+ <span>{{ form.file.vars.label|trans }}</span>
+ {{ form_widget(form.file) }}
+ </div>
+ <div class="file-path-wrapper">
+ <input class="file-path validate" type="text">
+ </div>
+ </div>
+ <div class="input-field col s6 with-checkbox">
+ <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
+ {{ form_widget(form.mark_as_read) }}
+ {{ form_label(form.mark_as_read) }}
+ </div>
+ </div>
+
+ {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
+
+ {{ form_rest(form) }}
+ </form>
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
+{% endblock %}
$crawler = $client->request('GET', '/import/');
$this->assertEquals(200, $client->getResponse()->getStatusCode());
- $this->assertEquals(6, $crawler->filter('blockquote')->count());
+ $this->assertEquals(7, $crawler->filter('blockquote')->count());
}
}
--- /dev/null
+<?php
+
+namespace Tests\Wallabag\ImportBundle\Import;
+
+use Wallabag\ImportBundle\Import\InstapaperImport;
+use Wallabag\UserBundle\Entity\User;
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\ImportBundle\Redis\Producer;
+use Monolog\Logger;
+use Monolog\Handler\TestHandler;
+use Simpleue\Queue\RedisQueue;
+use M6Web\Component\RedisMock\RedisMockFactory;
+
+class InstapaperImportTest extends \PHPUnit_Framework_TestCase
+{
+ protected $user;
+ protected $em;
+ protected $logHandler;
+ protected $contentProxy;
+
+ private function getInstapaperImport($unsetUser = false)
+ {
+ $this->user = new User();
+
+ $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $import = new InstapaperImport($this->em, $this->contentProxy);
+
+ $this->logHandler = new TestHandler();
+ $logger = new Logger('test', [$this->logHandler]);
+ $import->setLogger($logger);
+
+ if (false === $unsetUser) {
+ $import->setUser($this->user);
+ }
+
+ return $import;
+ }
+
+ public function testInit()
+ {
+ $instapaperImport = $this->getInstapaperImport();
+
+ $this->assertEquals('Instapaper', $instapaperImport->getName());
+ $this->assertNotEmpty($instapaperImport->getUrl());
+ $this->assertEquals('import.instapaper.description', $instapaperImport->getDescription());
+ }
+
+ public function testImport()
+ {
+ $instapaperImport = $this->getInstapaperImport();
+ $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->exactly(3))
+ ->method('findByUrlAndUserId')
+ ->willReturn(false);
+
+ $this->em
+ ->expects($this->any())
+ ->method('getRepository')
+ ->willReturn($entryRepo);
+
+ $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->exactly(3))
+ ->method('updateEntry')
+ ->willReturn($entry);
+
+ $res = $instapaperImport->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 3, 'queued' => 0], $instapaperImport->getSummary());
+ }
+
+ public function testImportAndMarkAllAsRead()
+ {
+ $instapaperImport = $this->getInstapaperImport();
+ $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->exactly(3))
+ ->method('findByUrlAndUserId')
+ ->will($this->onConsecutiveCalls(false, true, true));
+
+ $this->em
+ ->expects($this->any())
+ ->method('getRepository')
+ ->willReturn($entryRepo);
+
+ $this->contentProxy
+ ->expects($this->once())
+ ->method('updateEntry')
+ ->willReturn(new Entry($this->user));
+
+ // check that every entry persisted are archived
+ $this->em
+ ->expects($this->once())
+ ->method('persist')
+ ->with($this->callback(function ($persistedEntry) {
+ return $persistedEntry->isArchived();
+ }));
+
+ $res = $instapaperImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+
+ $this->assertEquals(['skipped' => 2, 'imported' => 1, 'queued' => 0], $instapaperImport->getSummary());
+ }
+
+ public function testImportWithRabbit()
+ {
+ $instapaperImport = $this->getInstapaperImport();
+ $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $producer
+ ->expects($this->exactly(3))
+ ->method('publish');
+
+ $instapaperImport->setProducer($producer);
+
+ $res = $instapaperImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 3], $instapaperImport->getSummary());
+ }
+
+ public function testImportWithRedis()
+ {
+ $instapaperImport = $this->getInstapaperImport();
+ $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
+
+ $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $entryRepo->expects($this->never())
+ ->method('findByUrlAndUserId');
+
+ $this->em
+ ->expects($this->never())
+ ->method('getRepository');
+
+ $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry')
+ ->disableOriginalConstructor()
+ ->getMock();
+
+ $this->contentProxy
+ ->expects($this->never())
+ ->method('updateEntry');
+
+ $factory = new RedisMockFactory();
+ $redisMock = $factory->getAdapter('Predis\Client', true);
+
+ $queue = new RedisQueue($redisMock, 'instapaper');
+ $producer = new Producer($queue);
+
+ $instapaperImport->setProducer($producer);
+
+ $res = $instapaperImport->setMarkAsRead(true)->import();
+
+ $this->assertTrue($res);
+ $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 3], $instapaperImport->getSummary());
+
+ $this->assertNotEmpty($redisMock->lpop('instapaper'));
+ }
+
+ public function testImportBadFile()
+ {
+ $instapaperImport = $this->getInstapaperImport();
+ $instapaperImport->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx');
+
+ $res = $instapaperImport->import();
+
+ $this->assertFalse($res);
+
+ $records = $this->logHandler->getRecords();
+ $this->assertContains('InstapaperImport: unable to read file', $records[0]['message']);
+ $this->assertEquals('ERROR', $records[0]['level_name']);
+ }
+
+ public function testImportUserNotDefined()
+ {
+ $instapaperImport = $this->getInstapaperImport(true);
+ $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv');
+
+ $res = $instapaperImport->import();
+
+ $this->assertFalse($res);
+
+ $records = $this->logHandler->getRecords();
+ $this->assertContains('InstapaperImport: user is not defined', $records[0]['message']);
+ $this->assertEquals('ERROR', $records[0]['level_name']);
+ }
+}
--- /dev/null
+URL,Title,Selection,Folder
+http://www.liberation.fr/societe/2012/12/06/baumettes-un-tour-en-cellule_865551,Baumettes : un tour en cellule,,Unread
+https://redditblog.com/2016/09/20/amp-and-reactredux/,AMP and React+Redux: Why Not?,,Archive
+https://medium.com/@the_minh/why-foursquare-swarm-is-still-my-favourite-social-network-e38228493e6c,Why Foursquare / Swarm is still my favourite social network,,Starred
projects / github / wallabag / wallabag.git / commitdiff
