aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/Wallabag/ImportBundle
diff options
context:
space:
mode:
Diffstat (limited to 'src/Wallabag/ImportBundle')
-rw-r--r--src/Wallabag/ImportBundle/Command/ImportCommand.php29
-rw-r--r--src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php44
-rw-r--r--src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php17
-rw-r--r--src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php74
-rw-r--r--src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php29
-rw-r--r--src/Wallabag/ImportBundle/Controller/BrowserController.php90
-rw-r--r--src/Wallabag/ImportBundle/Controller/ChromeController.php41
-rw-r--r--src/Wallabag/ImportBundle/Controller/FirefoxController.php41
-rw-r--r--src/Wallabag/ImportBundle/Controller/ImportController.php78
-rw-r--r--src/Wallabag/ImportBundle/Controller/InstapaperController.php77
-rw-r--r--src/Wallabag/ImportBundle/Controller/PocketController.php35
-rw-r--r--src/Wallabag/ImportBundle/Controller/ReadabilityController.php77
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagController.php10
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php10
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php10
-rw-r--r--src/Wallabag/ImportBundle/Form/Type/UploadImportType.php1
-rw-r--r--src/Wallabag/ImportBundle/Import/AbstractImport.php148
-rw-r--r--src/Wallabag/ImportBundle/Import/BrowserImport.php207
-rw-r--r--src/Wallabag/ImportBundle/Import/ChromeImport.php53
-rw-r--r--src/Wallabag/ImportBundle/Import/FirefoxImport.php53
-rw-r--r--src/Wallabag/ImportBundle/Import/InstapaperImport.php140
-rw-r--r--src/Wallabag/ImportBundle/Import/PocketImport.php219
-rw-r--r--src/Wallabag/ImportBundle/Import/ReadabilityImport.php134
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagImport.php134
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV1Import.php15
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV2Import.php14
-rw-r--r--src/Wallabag/ImportBundle/Redis/Producer.php36
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/rabbit.yml51
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/redis.yml141
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/services.yml44
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig43
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig43
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig8
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig23
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig2
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig45
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig12
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig45
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig2
39 files changed, 2042 insertions, 233 deletions
diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php
index dfbfc2f7..1df38295 100644
--- a/src/Wallabag/ImportBundle/Command/ImportCommand.php
+++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php
@@ -13,10 +13,12 @@ class ImportCommand extends ContainerAwareCommand
13 protected function configure() 13 protected function configure()
14 { 14 {
15 $this 15 $this
16 ->setName('wallabag:import-v1') 16 ->setName('wallabag:import')
17 ->setDescription('Import entries from a JSON export from a wallabag v1 instance') 17 ->setDescription('Import entries from a JSON export from a wallabag v1 instance')
18 ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') 18 ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate')
19 ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') 19 ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file')
20 ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1')
21 ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false)
20 ; 22 ;
21 } 23 }
22 24
@@ -24,6 +26,10 @@ class ImportCommand extends ContainerAwareCommand
24 { 26 {
25 $output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---'); 27 $output->writeln('Start : '.(new \DateTime())->format('d-m-Y G:i:s').' ---');
26 28
29 if (!file_exists($input->getArgument('filepath'))) {
30 throw new Exception(sprintf('File "%s" not found', $input->getArgument('filepath')));
31 }
32
27 $em = $this->getContainer()->get('doctrine')->getManager(); 33 $em = $this->getContainer()->get('doctrine')->getManager();
28 // Turning off doctrine default logs queries for saving memory 34 // Turning off doctrine default logs queries for saving memory
29 $em->getConnection()->getConfiguration()->setSQLLogger(null); 35 $em->getConnection()->getConfiguration()->setSQLLogger(null);
@@ -34,9 +40,26 @@ class ImportCommand extends ContainerAwareCommand
34 throw new Exception(sprintf('User with id "%s" not found', $input->getArgument('userId'))); 40 throw new Exception(sprintf('User with id "%s" not found', $input->getArgument('userId')));
35 } 41 }
36 42
37 $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); 43 switch ($input->getOption('importer')) {
44 case 'v2':
45 $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import');
46 break;
47 case 'firefox':
48 $wallabag = $this->getContainer()->get('wallabag_import.firefox.import');
49 break;
50 case 'chrome':
51 $wallabag = $this->getContainer()->get('wallabag_import.chrome.import');
52 break;
53 case 'v1':
54 default:
55 $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import');
56 break;
57 }
58
59 $wallabag->setMarkAsRead($input->getOption('markAsRead'));
60 $wallabag->setUser($user);
61
38 $res = $wallabag 62 $res = $wallabag
39 ->setUser($user)
40 ->setFilepath($input->getArgument('filepath')) 63 ->setFilepath($input->getArgument('filepath'))
41 ->import(); 64 ->import();
42 65
diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php
new file mode 100644
index 00000000..c2c11f11
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php
@@ -0,0 +1,44 @@
1<?php
2
3namespace Wallabag\ImportBundle\Command;
4
5use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
6use Symfony\Component\Config\Definition\Exception\Exception;
7use Symfony\Component\Console\Input\InputArgument;
8use Symfony\Component\Console\Input\InputOption;
9use Symfony\Component\Console\Input\InputInterface;
10use Symfony\Component\Console\Output\OutputInterface;
11use Simpleue\Worker\QueueWorker;
12
13class RedisWorkerCommand extends ContainerAwareCommand
14{
15 protected function configure()
16 {
17 $this
18 ->setName('wallabag:import:redis-worker')
19 ->setDescription('Launch Redis worker')
20 ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, firefox, chrome or instapaper')
21 ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false)
22 ;
23 }
24
25 protected function execute(InputInterface $input, OutputInterface $output)
26 {
27 $output->writeln('Worker started at: '.(new \DateTime())->format('d-m-Y G:i:s'));
28 $output->writeln('Waiting for message ...');
29
30 $serviceName = $input->getArgument('serviceName');
31
32 if (!$this->getContainer()->has('wallabag_import.queue.redis.'.$serviceName) || !$this->getContainer()->has('wallabag_import.consumer.redis.'.$serviceName)) {
33 throw new Exception(sprintf('No queue or consumer found for service name: "%s"', $input->getArgument('serviceName')));
34 }
35
36 $worker = new QueueWorker(
37 $this->getContainer()->get('wallabag_import.queue.redis.'.$serviceName),
38 $this->getContainer()->get('wallabag_import.consumer.redis.'.$serviceName),
39 $input->getOption('maxIterations')
40 );
41
42 $worker->start();
43 }
44}
diff --git a/src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php
new file mode 100644
index 00000000..5aaa8c03
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Consumer/AMQPEntryConsumer.php
@@ -0,0 +1,17 @@
1<?php
2
3namespace Wallabag\ImportBundle\Consumer;
4
5use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
6use PhpAmqpLib\Message\AMQPMessage;
7
8class AMQPEntryConsumer extends AbstractConsumer implements ConsumerInterface
9{
10 /**
11 * {@inheritdoc}
12 */
13 public function execute(AMQPMessage $msg)
14 {
15 return $this->handleMessage($msg->body);
16 }
17}
diff --git a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php
new file mode 100644
index 00000000..2b85ad76
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php
@@ -0,0 +1,74 @@
1<?php
2
3namespace Wallabag\ImportBundle\Consumer;
4
5use Doctrine\ORM\EntityManager;
6use Wallabag\ImportBundle\Import\AbstractImport;
7use Wallabag\UserBundle\Repository\UserRepository;
8use Wallabag\CoreBundle\Entity\Entry;
9use Wallabag\CoreBundle\Entity\Tag;
10use Psr\Log\LoggerInterface;
11use Psr\Log\NullLogger;
12
13abstract class AbstractConsumer
14{
15 protected $em;
16 protected $userRepository;
17 protected $import;
18 protected $logger;
19
20 public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
21 {
22 $this->em = $em;
23 $this->userRepository = $userRepository;
24 $this->import = $import;
25 $this->logger = $logger ?: new NullLogger();
26 }
27
28 /**
29 * Handle a message and save it.
30 *
31 * @param string $body Message from the queue (in json)
32 *
33 * @return bool
34 */
35 protected function handleMessage($body)
36 {
37 $storedEntry = json_decode($body, true);
38
39 $user = $this->userRepository->find($storedEntry['userId']);
40
41 // no user? Drop message
42 if (null === $user) {
43 $this->logger->warning('Unable to retrieve user', ['entry' => $storedEntry]);
44
45 return false;
46 }
47
48 $this->import->setUser($user);
49
50 $entry = $this->import->parseEntry($storedEntry);
51
52 if (null === $entry) {
53 $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
54
55 return false;
56 }
57
58 try {
59 $this->em->flush();
60
61 // clear only affected entities
62 $this->em->clear(Entry::class);
63 $this->em->clear(Tag::class);
64 } catch (\Exception $e) {
65 $this->logger->warning('Unable to save entry', ['entry' => $storedEntry, 'exception' => $e]);
66
67 return false;
68 }
69
70 $this->logger->info('Content with url imported! ('.$entry->getUrl().')');
71
72 return true;
73 }
74}
diff --git a/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php
new file mode 100644
index 00000000..450b71ff
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Consumer/RedisEntryConsumer.php
@@ -0,0 +1,29 @@
1<?php
2
3namespace Wallabag\ImportBundle\Consumer;
4
5use Simpleue\Job\Job;
6
7class RedisEntryConsumer extends AbstractConsumer implements Job
8{
9 /**
10 * Handle one message by one message.
11 *
12 * @param string $job Content of the message (directly from Redis)
13 *
14 * @return bool
15 */
16 public function manage($job)
17 {
18 return $this->handleMessage($job);
19 }
20
21 /**
22 * Should tell if the given job will kill the worker.
23 * We don't want to stop it :).
24 */
25 public function isStopJob($job)
26 {
27 return false;
28 }
29}
diff --git a/src/Wallabag/ImportBundle/Controller/BrowserController.php b/src/Wallabag/ImportBundle/Controller/BrowserController.php
new file mode 100644
index 00000000..144a4880
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/BrowserController.php
@@ -0,0 +1,90 @@
1<?php
2
3namespace Wallabag\ImportBundle\Controller;
4
5use Symfony\Bundle\FrameworkBundle\Controller\Controller;
6use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
7use Symfony\Component\HttpFoundation\Request;
8use Symfony\Component\HttpFoundation\Response;
9use Wallabag\ImportBundle\Form\Type\UploadImportType;
10
11abstract class BrowserController extends Controller
12{
13 /**
14 * Return the service to handle the import.
15 *
16 * @return \Wallabag\ImportBundle\Import\ImportInterface
17 */
18 abstract protected function getImportService();
19
20 /**
21 * Return the template used for the form.
22 *
23 * @return string
24 */
25 abstract protected function getImportTemplate();
26
27 /**
28 * @Route("/browser", name="import_browser")
29 *
30 * @param Request $request
31 *
32 * @return Response
33 */
34 public function indexAction(Request $request)
35 {
36 $form = $this->createForm(UploadImportType::class);
37 $form->handleRequest($request);
38
39 $wallabag = $this->getImportService();
40 $wallabag->setUser($this->getUser());
41
42 if ($form->isValid()) {
43 $file = $form->get('file')->getData();
44 $markAsRead = $form->get('mark_as_read')->getData();
45 $name = $this->getUser()->getId().'.json';
46
47 if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
48 $res = $wallabag
49 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
50 ->setMarkAsRead($markAsRead)
51 ->import();
52
53 $message = 'flashes.import.notice.failed';
54
55 if (true === $res) {
56 $summary = $wallabag->getSummary();
57 $message = $this->get('translator')->trans('flashes.import.notice.summary', [
58 '%imported%' => $summary['imported'],
59 '%skipped%' => $summary['skipped'],
60 ]);
61
62 if (0 < $summary['queued']) {
63 $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
64 '%queued%' => $summary['queued'],
65 ]);
66 }
67
68 unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
69 }
70
71 $this->get('session')->getFlashBag()->add(
72 'notice',
73 $message
74 );
75
76 return $this->redirect($this->generateUrl('homepage'));
77 } else {
78 $this->get('session')->getFlashBag()->add(
79 'notice',
80 'flashes.import.notice.failed_on_file'
81 );
82 }
83 }
84
85 return $this->render($this->getImportTemplate(), [
86 'form' => $form->createView(),
87 'import' => $wallabag,
88 ]);
89 }
90}
diff --git a/src/Wallabag/ImportBundle/Controller/ChromeController.php b/src/Wallabag/ImportBundle/Controller/ChromeController.php
new file mode 100644
index 00000000..454f3347
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/ChromeController.php
@@ -0,0 +1,41 @@
1<?php
2
3namespace Wallabag\ImportBundle\Controller;
4
5use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
6use Symfony\Component\HttpFoundation\Request;
7
8class ChromeController extends BrowserController
9{
10 /**
11 * {@inheritdoc}
12 */
13 protected function getImportService()
14 {
15 $service = $this->get('wallabag_import.chrome.import');
16
17 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
18 $service->setProducer($this->get('old_sound_rabbit_mq.import_chrome_producer'));
19 } elseif ($this->get('craue_config')->get('import_with_redis')) {
20 $service->setProducer($this->get('wallabag_import.producer.redis.chrome'));
21 }
22
23 return $service;
24 }
25
26 /**
27 * {@inheritdoc}
28 */
29 protected function getImportTemplate()
30 {
31 return 'WallabagImportBundle:Chrome:index.html.twig';
32 }
33
34 /**
35 * @Route("/chrome", name="import_chrome")
36 */
37 public function indexAction(Request $request)
38 {
39 return parent::indexAction($request);
40 }
41}
diff --git a/src/Wallabag/ImportBundle/Controller/FirefoxController.php b/src/Wallabag/ImportBundle/Controller/FirefoxController.php
new file mode 100644
index 00000000..c329b9c4
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/FirefoxController.php
@@ -0,0 +1,41 @@
1<?php
2
3namespace Wallabag\ImportBundle\Controller;
4
5use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
6use Symfony\Component\HttpFoundation\Request;
7
8class FirefoxController extends BrowserController
9{
10 /**
11 * {@inheritdoc}
12 */
13 protected function getImportService()
14 {
15 $service = $this->get('wallabag_import.firefox.import');
16
17 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
18 $service->setProducer($this->get('old_sound_rabbit_mq.import_firefox_producer'));
19 } elseif ($this->get('craue_config')->get('import_with_redis')) {
20 $service->setProducer($this->get('wallabag_import.producer.redis.firefox'));
21 }
22
23 return $service;
24 }
25
26 /**
27 * {@inheritdoc}
28 */
29 protected function getImportTemplate()
30 {
31 return 'WallabagImportBundle:Firefox:index.html.twig';
32 }
33
34 /**
35 * @Route("/firefox", name="import_firefox")
36 */
37 public function indexAction(Request $request)
38 {
39 return parent::indexAction($request);
40 }
41}
diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php
index c1486e38..15de75ff 100644
--- a/src/Wallabag/ImportBundle/Controller/ImportController.php
+++ b/src/Wallabag/ImportBundle/Controller/ImportController.php
@@ -16,4 +16,82 @@ class ImportController extends Controller
16 'imports' => $this->get('wallabag_import.chain')->getAll(), 16 'imports' => $this->get('wallabag_import.chain')->getAll(),
17 ]); 17 ]);
18 } 18 }
19
20 /**
21 * Display how many messages are queue (both in Redis and RabbitMQ).
22 * Only for admins.
23 */
24 public function checkQueueAction()
25 {
26 $nbRedisMessages = null;
27 $nbRabbitMessages = null;
28 $redisNotInstalled = false;
29 $rabbitNotInstalled = false;
30
31 if (!$this->get('security.authorization_checker')->isGranted('ROLE_SUPER_ADMIN')) {
32 return $this->render('WallabagImportBundle:Import:check_queue.html.twig');
33 }
34
35 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
36 // in case rabbit is activated but not installed
37 try {
38 $nbRabbitMessages = $this->getTotalMessageInRabbitQueue('pocket')
39 + $this->getTotalMessageInRabbitQueue('readability')
40 + $this->getTotalMessageInRabbitQueue('wallabag_v1')
41 + $this->getTotalMessageInRabbitQueue('wallabag_v2')
42 + $this->getTotalMessageInRabbitQueue('firefox')
43 + $this->getTotalMessageInRabbitQueue('chrome')
44 + $this->getTotalMessageInRabbitQueue('instapaper')
45 ;
46 } catch (\Exception $e) {
47 $rabbitNotInstalled = true;
48 }
49 } elseif ($this->get('craue_config')->get('import_with_redis')) {
50 $redis = $this->get('wallabag_core.redis.client');
51
52 try {
53 $nbRedisMessages = $redis->llen('wallabag.import.pocket')
54 + $redis->llen('wallabag.import.readability')
55 + $redis->llen('wallabag.import.wallabag_v1')
56 + $redis->llen('wallabag.import.wallabag_v2')
57 + $redis->llen('wallabag.import.firefox')
58 + $redis->llen('wallabag.import.chrome')
59 + $redis->llen('wallabag.import.instapaper')
60 ;
61 } catch (\Exception $e) {
62 $redisNotInstalled = true;
63 }
64 }
65
66 return $this->render('WallabagImportBundle:Import:check_queue.html.twig', [
67 'nbRedisMessages' => $nbRedisMessages,
68 'nbRabbitMessages' => $nbRabbitMessages,
69 'redisNotInstalled' => $redisNotInstalled,
70 'rabbitNotInstalled' => $rabbitNotInstalled,
71 ]);
72 }
73
74 /**
75 * Count message in RabbitMQ queue.
76 * It get one message without acking it (so it'll stay in the queue)
77 * which will include the total of *other* messages in the queue.
78 * Adding one to that messages will result in the full total message.
79 *
80 * @param string $importService The import service related: pocket, readability, wallabag_v1 or wallabag_v2
81 *
82 * @return int
83 */
84 private function getTotalMessageInRabbitQueue($importService)
85 {
86 $message = $this
87 ->get('old_sound_rabbit_mq.import_'.$importService.'_consumer')
88 ->getChannel()
89 ->basic_get('wallabag.import.'.$importService);
90
91 if (null === $message) {
92 return 0;
93 }
94
95 return $message->delivery_info['message_count'] + 1;
96 }
19} 97}
diff --git a/src/Wallabag/ImportBundle/Controller/InstapaperController.php b/src/Wallabag/ImportBundle/Controller/InstapaperController.php
new file mode 100644
index 00000000..c3fc8a39
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/InstapaperController.php
@@ -0,0 +1,77 @@
1<?php
2
3namespace Wallabag\ImportBundle\Controller;
4
5use Symfony\Bundle\FrameworkBundle\Controller\Controller;
6use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
7use Symfony\Component\HttpFoundation\Request;
8use Wallabag\ImportBundle\Form\Type\UploadImportType;
9
10class InstapaperController extends Controller
11{
12 /**
13 * @Route("/instapaper", name="import_instapaper")
14 */
15 public function indexAction(Request $request)
16 {
17 $form = $this->createForm(UploadImportType::class);
18 $form->handleRequest($request);
19
20 $instapaper = $this->get('wallabag_import.instapaper.import');
21 $instapaper->setUser($this->getUser());
22
23 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
24 $instapaper->setProducer($this->get('old_sound_rabbit_mq.import_instapaper_producer'));
25 } elseif ($this->get('craue_config')->get('import_with_redis')) {
26 $instapaper->setProducer($this->get('wallabag_import.producer.redis.instapaper'));
27 }
28
29 if ($form->isValid()) {
30 $file = $form->get('file')->getData();
31 $markAsRead = $form->get('mark_as_read')->getData();
32 $name = 'instapaper_'.$this->getUser()->getId().'.csv';
33
34 if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
35 $res = $instapaper
36 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
37 ->setMarkAsRead($markAsRead)
38 ->import();
39
40 $message = 'flashes.import.notice.failed';
41
42 if (true === $res) {
43 $summary = $instapaper->getSummary();
44 $message = $this->get('translator')->trans('flashes.import.notice.summary', [
45 '%imported%' => $summary['imported'],
46 '%skipped%' => $summary['skipped'],
47 ]);
48
49 if (0 < $summary['queued']) {
50 $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
51 '%queued%' => $summary['queued'],
52 ]);
53 }
54
55 unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
56 }
57
58 $this->get('session')->getFlashBag()->add(
59 'notice',
60 $message
61 );
62
63 return $this->redirect($this->generateUrl('homepage'));
64 } else {
65 $this->get('session')->getFlashBag()->add(
66 'notice',
67 'flashes.import.notice.failed_on_file'
68 );
69 }
70 }
71
72 return $this->render('WallabagImportBundle:Instapaper:index.html.twig', [
73 'form' => $form->createView(),
74 'import' => $instapaper,
75 ]);
76 }
77}
diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php
index 36ee25bf..56be5cbf 100644
--- a/src/Wallabag/ImportBundle/Controller/PocketController.php
+++ b/src/Wallabag/ImportBundle/Controller/PocketController.php
@@ -11,11 +11,30 @@ use Symfony\Component\Form\Extension\Core\Type\CheckboxType;
11class PocketController extends Controller 11class PocketController extends Controller
12{ 12{
13 /** 13 /**
14 * Return Pocket Import Service with or without RabbitMQ enabled.
15 *
16 * @return \Wallabag\ImportBundle\Import\PocketImport
17 */
18 private function getPocketImportService()
19 {
20 $pocket = $this->get('wallabag_import.pocket.import');
21 $pocket->setUser($this->getUser());
22
23 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
24 $pocket->setProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
25 } elseif ($this->get('craue_config')->get('import_with_redis')) {
26 $pocket->setProducer($this->get('wallabag_import.producer.redis.pocket'));
27 }
28
29 return $pocket;
30 }
31
32 /**
14 * @Route("/pocket", name="import_pocket") 33 * @Route("/pocket", name="import_pocket")
15 */ 34 */
16 public function indexAction() 35 public function indexAction()
17 { 36 {
18 $pocket = $this->get('wallabag_import.pocket.import'); 37 $pocket = $this->getPocketImportService();
19 $form = $this->createFormBuilder($pocket) 38 $form = $this->createFormBuilder($pocket)
20 ->add('mark_as_read', CheckboxType::class, [ 39 ->add('mark_as_read', CheckboxType::class, [
21 'label' => 'import.form.mark_as_read_label', 40 'label' => 'import.form.mark_as_read_label',
@@ -24,8 +43,8 @@ class PocketController extends Controller
24 ->getForm(); 43 ->getForm();
25 44
26 return $this->render('WallabagImportBundle:Pocket:index.html.twig', [ 45 return $this->render('WallabagImportBundle:Pocket:index.html.twig', [
27 'import' => $this->get('wallabag_import.pocket.import'), 46 'import' => $this->getPocketImportService(),
28 'has_consumer_key' => '' == trim($this->get('craue_config')->get('pocket_consumer_key')) ? false : true, 47 'has_consumer_key' => '' === trim($this->getUser()->getConfig()->getPocketConsumerKey()) ? false : true,
29 'form' => $form->createView(), 48 'form' => $form->createView(),
30 ]); 49 ]);
31 } 50 }
@@ -35,7 +54,7 @@ class PocketController extends Controller
35 */ 54 */
36 public function authAction(Request $request) 55 public function authAction(Request $request)
37 { 56 {
38 $requestToken = $this->get('wallabag_import.pocket.import') 57 $requestToken = $this->getPocketImportService()
39 ->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL)); 58 ->getRequestToken($this->generateUrl('import', [], UrlGeneratorInterface::ABSOLUTE_URL));
40 59
41 if (false === $requestToken) { 60 if (false === $requestToken) {
@@ -62,7 +81,7 @@ class PocketController extends Controller
62 public function callbackAction() 81 public function callbackAction()
63 { 82 {
64 $message = 'flashes.import.notice.failed'; 83 $message = 'flashes.import.notice.failed';
65 $pocket = $this->get('wallabag_import.pocket.import'); 84 $pocket = $this->getPocketImportService();
66 85
67 $markAsRead = $this->get('session')->get('mark_as_read'); 86 $markAsRead = $this->get('session')->get('mark_as_read');
68 $this->get('session')->remove('mark_as_read'); 87 $this->get('session')->remove('mark_as_read');
@@ -83,6 +102,12 @@ class PocketController extends Controller
83 '%imported%' => $summary['imported'], 102 '%imported%' => $summary['imported'],
84 '%skipped%' => $summary['skipped'], 103 '%skipped%' => $summary['skipped'],
85 ]); 104 ]);
105
106 if (0 < $summary['queued']) {
107 $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
108 '%queued%' => $summary['queued'],
109 ]);
110 }
86 } 111 }
87 112
88 $this->get('session')->getFlashBag()->add( 113 $this->get('session')->getFlashBag()->add(
diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
new file mode 100644
index 00000000..d00e22c2
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
@@ -0,0 +1,77 @@
1<?php
2
3namespace Wallabag\ImportBundle\Controller;
4
5use Symfony\Bundle\FrameworkBundle\Controller\Controller;
6use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
7use Symfony\Component\HttpFoundation\Request;
8use Wallabag\ImportBundle\Form\Type\UploadImportType;
9
10class ReadabilityController extends Controller
11{
12 /**
13 * @Route("/readability", name="import_readability")
14 */
15 public function indexAction(Request $request)
16 {
17 $form = $this->createForm(UploadImportType::class);
18 $form->handleRequest($request);
19
20 $readability = $this->get('wallabag_import.readability.import');
21 $readability->setUser($this->getUser());
22
23 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
24 $readability->setProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
25 } elseif ($this->get('craue_config')->get('import_with_redis')) {
26 $readability->setProducer($this->get('wallabag_import.producer.redis.readability'));
27 }
28
29 if ($form->isValid()) {
30 $file = $form->get('file')->getData();
31 $markAsRead = $form->get('mark_as_read')->getData();
32 $name = 'readability_'.$this->getUser()->getId().'.json';
33
34 if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
35 $res = $readability
36 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
37 ->setMarkAsRead($markAsRead)
38 ->import();
39
40 $message = 'flashes.import.notice.failed';
41
42 if (true === $res) {
43 $summary = $readability->getSummary();
44 $message = $this->get('translator')->trans('flashes.import.notice.summary', [
45 '%imported%' => $summary['imported'],
46 '%skipped%' => $summary['skipped'],
47 ]);
48
49 if (0 < $summary['queued']) {
50 $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
51 '%queued%' => $summary['queued'],
52 ]);
53 }
54
55 unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
56 }
57
58 $this->get('session')->getFlashBag()->add(
59 'notice',
60 $message
61 );
62
63 return $this->redirect($this->generateUrl('homepage'));
64 } else {
65 $this->get('session')->getFlashBag()->add(
66 'notice',
67 'flashes.import.notice.failed_on_file'
68 );
69 }
70 }
71
72 return $this->render('WallabagImportBundle:Readability:index.html.twig', [
73 'form' => $form->createView(),
74 'import' => $readability,
75 ]);
76 }
77}
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php
index 76ced0d2..9c0cde80 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagController.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php
@@ -38,15 +38,15 @@ abstract class WallabagController extends Controller
38 $form->handleRequest($request); 38 $form->handleRequest($request);
39 39
40 $wallabag = $this->getImportService(); 40 $wallabag = $this->getImportService();
41 $wallabag->setUser($this->getUser());
41 42
42 if ($form->isValid()) { 43 if ($form->isValid()) {
43 $file = $form->get('file')->getData(); 44 $file = $form->get('file')->getData();
44 $markAsRead = $form->get('mark_as_read')->getData(); 45 $markAsRead = $form->get('mark_as_read')->getData();
45 $name = $this->getUser()->getId().'.json'; 46 $name = $this->getUser()->getId().'.json';
46 47
47 if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { 48 if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
48 $res = $wallabag 49 $res = $wallabag
49 ->setUser($this->getUser())
50 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) 50 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
51 ->setMarkAsRead($markAsRead) 51 ->setMarkAsRead($markAsRead)
52 ->import(); 52 ->import();
@@ -60,6 +60,12 @@ abstract class WallabagController extends Controller
60 '%skipped%' => $summary['skipped'], 60 '%skipped%' => $summary['skipped'],
61 ]); 61 ]);
62 62
63 if (0 < $summary['queued']) {
64 $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [
65 '%queued%' => $summary['queued'],
66 ]);
67 }
68
63 unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); 69 unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
64 } 70 }
65 71
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
index 3e748d57..312c7a35 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
@@ -12,7 +12,15 @@ class WallabagV1Controller extends WallabagController
12 */ 12 */
13 protected function getImportService() 13 protected function getImportService()
14 { 14 {
15 return $this->get('wallabag_import.wallabag_v1.import'); 15 $service = $this->get('wallabag_import.wallabag_v1.import');
16
17 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
18 $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
19 } elseif ($this->get('craue_config')->get('import_with_redis')) {
20 $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1'));
21 }
22
23 return $service;
16 } 24 }
17 25
18 /** 26 /**
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
index c2a42165..45211fe6 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
@@ -12,7 +12,15 @@ class WallabagV2Controller extends WallabagController
12 */ 12 */
13 protected function getImportService() 13 protected function getImportService()
14 { 14 {
15 return $this->get('wallabag_import.wallabag_v2.import'); 15 $service = $this->get('wallabag_import.wallabag_v2.import');
16
17 if ($this->get('craue_config')->get('import_with_rabbitmq')) {
18 $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
19 } elseif ($this->get('craue_config')->get('import_with_redis')) {
20 $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v2'));
21 }
22
23 return $service;
16 } 24 }
17 25
18 /** 26 /**
diff --git a/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php b/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php
index 92a167d9..f50424c1 100644
--- a/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php
+++ b/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php
@@ -15,6 +15,7 @@ class UploadImportType extends AbstractType
15 $builder 15 $builder
16 ->add('file', FileType::class, [ 16 ->add('file', FileType::class, [
17 'label' => 'import.form.file_label', 17 'label' => 'import.form.file_label',
18 'required' => true,
18 ]) 19 ])
19 ->add('mark_as_read', CheckboxType::class, [ 20 ->add('mark_as_read', CheckboxType::class, [
20 'label' => 'import.form.mark_as_read_label', 21 'label' => 'import.form.mark_as_read_label',
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index 14377a35..764b390a 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -7,12 +7,21 @@ use Psr\Log\NullLogger;
7use Doctrine\ORM\EntityManager; 7use Doctrine\ORM\EntityManager;
8use Wallabag\CoreBundle\Helper\ContentProxy; 8use Wallabag\CoreBundle\Helper\ContentProxy;
9use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
10use Wallabag\CoreBundle\Entity\Tag;
11use Wallabag\UserBundle\Entity\User;
12use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
10 13
11abstract class AbstractImport implements ImportInterface 14abstract class AbstractImport implements ImportInterface
12{ 15{
13 protected $em; 16 protected $em;
14 protected $logger; 17 protected $logger;
15 protected $contentProxy; 18 protected $contentProxy;
19 protected $producer;
20 protected $user;
21 protected $markAsRead;
22 protected $skippedEntries = 0;
23 protected $importedEntries = 0;
24 protected $queuedEntries = 0;
16 25
17 public function __construct(EntityManager $em, ContentProxy $contentProxy) 26 public function __construct(EntityManager $em, ContentProxy $contentProxy)
18 { 27 {
@@ -27,21 +36,154 @@ abstract class AbstractImport implements ImportInterface
27 } 36 }
28 37
29 /** 38 /**
39 * Set RabbitMQ/Redis Producer to send each entry to a queue.
40 * This method should be called when user has enabled RabbitMQ.
41 *
42 * @param ProducerInterface $producer
43 */
44 public function setProducer(ProducerInterface $producer)
45 {
46 $this->producer = $producer;
47 }
48
49 /**
50 * Set current user.
51 * Could the current *connected* user or one retrieve by the consumer.
52 *
53 * @param User $user
54 */
55 public function setUser(User $user)
56 {
57 $this->user = $user;
58 }
59
60 /**
61 * Set whether articles must be all marked as read.
62 *
63 * @param bool $markAsRead
64 */
65 public function setMarkAsRead($markAsRead)
66 {
67 $this->markAsRead = $markAsRead;
68
69 return $this;
70 }
71
72 /**
73 * Get whether articles must be all marked as read.
74 */
75 public function getMarkAsRead()
76 {
77 return $this->markAsRead;
78 }
79
80 /**
30 * Fetch content from the ContentProxy (using graby). 81 * Fetch content from the ContentProxy (using graby).
31 * If it fails return false instead of the updated entry. 82 * If it fails return the given entry to be saved in all case (to avoid user to loose the content).
32 * 83 *
33 * @param Entry $entry Entry to update 84 * @param Entry $entry Entry to update
34 * @param string $url Url to grab content for 85 * @param string $url Url to grab content for
35 * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url 86 * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
36 * 87 *
37 * @return Entry|false 88 * @return Entry
38 */ 89 */
39 protected function fetchContent(Entry $entry, $url, array $content = []) 90 protected function fetchContent(Entry $entry, $url, array $content = [])
40 { 91 {
41 try { 92 try {
42 return $this->contentProxy->updateEntry($entry, $url, $content); 93 return $this->contentProxy->updateEntry($entry, $url, $content);
43 } catch (\Exception $e) { 94 } catch (\Exception $e) {
44 return false; 95 return $entry;
45 } 96 }
46 } 97 }
98
99 /**
100 * Parse and insert all given entries.
101 *
102 * @param $entries
103 */
104 protected function parseEntries($entries)
105 {
106 $i = 1;
107
108 foreach ($entries as $importedEntry) {
109 if ($this->markAsRead) {
110 $importedEntry = $this->setEntryAsRead($importedEntry);
111 }
112
113 $entry = $this->parseEntry($importedEntry);
114
115 if (null === $entry) {
116 continue;
117 }
118
119 // flush every 20 entries
120 if (($i % 20) === 0) {
121 $this->em->flush();
122
123 // clear only affected entities
124 $this->em->clear(Entry::class);
125 $this->em->clear(Tag::class);
126 }
127 ++$i;
128 }
129
130 $this->em->flush();
131 }
132
133 /**
134 * Parse entries and send them to the queue.
135 * It should just be a simple loop on all item, no call to the database should be done
136 * to speedup queuing.
137 *
138 * Faster parse entries for Producer.
139 * We don't care to make check at this time. They'll be done by the consumer.
140 *
141 * @param array $entries
142 */
143 protected function parseEntriesForProducer(array $entries)
144 {
145 foreach ($entries as $importedEntry) {
146 // set userId for the producer (it won't know which user is connected)
147 $importedEntry['userId'] = $this->user->getId();
148
149 if ($this->markAsRead) {
150 $importedEntry = $this->setEntryAsRead($importedEntry);
151 }
152
153 ++$this->queuedEntries;
154
155 $this->producer->publish(json_encode($importedEntry));
156 }
157 }
158
159 /**
160 * {@inheritdoc}
161 */
162 public function getSummary()
163 {
164 return [
165 'skipped' => $this->skippedEntries,
166 'imported' => $this->importedEntries,
167 'queued' => $this->queuedEntries,
168 ];
169 }
170
171 /**
172 * Parse one entry.
173 *
174 * @param array $importedEntry
175 *
176 * @return Entry
177 */
178 abstract public function parseEntry(array $importedEntry);
179
180 /**
181 * Set current imported entry to archived / read.
182 * Implementation is different accross all imports.
183 *
184 * @param array $importedEntry
185 *
186 * @return array
187 */
188 abstract protected function setEntryAsRead(array $importedEntry);
47} 189}
diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php
new file mode 100644
index 00000000..9d75685b
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php
@@ -0,0 +1,207 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5use Wallabag\CoreBundle\Entity\Entry;
6use Wallabag\UserBundle\Entity\User;
7use Wallabag\CoreBundle\Helper\ContentProxy;
8
9abstract class BrowserImport extends AbstractImport
10{
11 protected $filepath;
12
13 /**
14 * {@inheritdoc}
15 */
16 abstract public function getName();
17
18 /**
19 * {@inheritdoc}
20 */
21 abstract public function getUrl();
22
23 /**
24 * {@inheritdoc}
25 */
26 abstract public function getDescription();
27
28 /**
29 * {@inheritdoc}
30 */
31 public function import()
32 {
33 if (!$this->user) {
34 $this->logger->error('Wallabag Browser Import: user is not defined');
35
36 return false;
37 }
38
39 if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
40 $this->logger->error('Wallabag Browser Import: unable to read file', ['filepath' => $this->filepath]);
41
42 return false;
43 }
44
45 $data = json_decode(file_get_contents($this->filepath), true);
46
47 if (empty($data)) {
48 $this->logger->error('Wallabag Browser: no entries in imported file');
49
50 return false;
51 }
52
53 if ($this->producer) {
54 $this->parseEntriesForProducer($data);
55
56 return true;
57 }
58
59 $this->parseEntries($data);
60
61 return true;
62 }
63
64 /**
65 * Set file path to the json file.
66 *
67 * @param string $filepath
68 */
69 public function setFilepath($filepath)
70 {
71 $this->filepath = $filepath;
72
73 return $this;
74 }
75
76 /**
77 * Parse and insert all given entries.
78 *
79 * @param $entries
80 */
81 protected function parseEntries($entries)
82 {
83 $i = 1;
84
85 foreach ($entries as $importedEntry) {
86 if ((array) $importedEntry !== $importedEntry) {
87 continue;
88 }
89
90 $entry = $this->parseEntry($importedEntry);
91
92 if (null === $entry) {
93 continue;
94 }
95
96 // flush every 20 entries
97 if (($i % 20) === 0) {
98 $this->em->flush();
99 }
100 ++$i;
101 }
102
103 $this->em->flush();
104 }
105
106 /**
107 * Parse entries and send them to the queue.
108 * It should just be a simple loop on all item, no call to the database should be done
109 * to speedup queuing.
110 *
111 * Faster parse entries for Producer.
112 * We don't care to make check at this time. They'll be done by the consumer.
113 *
114 * @param array $entries
115 */
116 protected function parseEntriesForProducer(array $entries)
117 {
118 foreach ($entries as $importedEntry) {
119 if ((array) $importedEntry !== $importedEntry) {
120 continue;
121 }
122
123 // set userId for the producer (it won't know which user is connected)
124 $importedEntry['userId'] = $this->user->getId();
125
126 if ($this->markAsRead) {
127 $importedEntry = $this->setEntryAsRead($importedEntry);
128 }
129
130 ++$this->queuedEntries;
131
132 $this->producer->publish(json_encode($importedEntry));
133 }
134 }
135
136 /**
137 * {@inheritdoc}
138 */
139 public function parseEntry(array $importedEntry)
140 {
141 if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) {
142 $this->parseEntries($importedEntry);
143
144 return;
145 }
146
147 if (array_key_exists('children', $importedEntry)) {
148 $this->parseEntries($importedEntry['children']);
149
150 return;
151 }
152
153 if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) {
154 return;
155 }
156
157 $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url'];
158
159 $existingEntry = $this->em
160 ->getRepository('WallabagCoreBundle:Entry')
161 ->findByUrlAndUserId($url, $this->user->getId());
162
163 if (false !== $existingEntry) {
164 ++$this->skippedEntries;
165
166 return;
167 }
168
169 $data = $this->prepareEntry($importedEntry);
170
171 $entry = new Entry($this->user);
172 $entry->setUrl($data['url']);
173 $entry->setTitle($data['title']);
174
175 // update entry with content (in case fetching failed, the given entry will be return)
176 $entry = $this->fetchContent($entry, $data['url'], $data);
177
178 if (array_key_exists('tags', $data)) {
179 $this->contentProxy->assignTagsToEntry(
180 $entry,
181 $data['tags']
182 );
183 }
184
185 $entry->setArchived($data['is_archived']);
186
187 if (!empty($data['created_at'])) {
188 $dt = new \DateTime();
189 $entry->setCreatedAt($dt->setTimestamp($data['created_at']));
190 }
191
192 $this->em->persist($entry);
193 ++$this->importedEntries;
194
195 return $entry;
196 }
197
198 /**
199 * {@inheritdoc}
200 */
201 protected function setEntryAsRead(array $importedEntry)
202 {
203 $importedEntry['is_archived'] = 1;
204
205 return $importedEntry;
206 }
207}
diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php
new file mode 100644
index 00000000..d7620bcb
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php
@@ -0,0 +1,53 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5class ChromeImport extends BrowserImport
6{
7 protected $filepath;
8
9 /**
10 * {@inheritdoc}
11 */
12 public function getName()
13 {
14 return 'Chrome';
15 }
16
17 /**
18 * {@inheritdoc}
19 */
20 public function getUrl()
21 {
22 return 'import_chrome';
23 }
24
25 /**
26 * {@inheritdoc}
27 */
28 public function getDescription()
29 {
30 return 'import.chrome.description';
31 }
32
33 /**
34 * {@inheritdoc}
35 */
36 protected function prepareEntry(array $entry = [])
37 {
38 $data = [
39 'title' => $entry['name'],
40 'html' => '',
41 'url' => $entry['url'],
42 'is_archived' => $this->markAsRead,
43 'tags' => '',
44 'created_at' => substr($entry['date_added'], 0, 10),
45 ];
46
47 if (array_key_exists('tags', $entry) && $entry['tags'] != '') {
48 $data['tags'] = $entry['tags'];
49 }
50
51 return $data;
52 }
53}
diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php
new file mode 100644
index 00000000..e010f5a4
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php
@@ -0,0 +1,53 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5class FirefoxImport extends BrowserImport
6{
7 protected $filepath;
8
9 /**
10 * {@inheritdoc}
11 */
12 public function getName()
13 {
14 return 'Firefox';
15 }
16
17 /**
18 * {@inheritdoc}
19 */
20 public function getUrl()
21 {
22 return 'import_firefox';
23 }
24
25 /**
26 * {@inheritdoc}
27 */
28 public function getDescription()
29 {
30 return 'import.firefox.description';
31 }
32
33 /**
34 * {@inheritdoc}
35 */
36 protected function prepareEntry(array $entry = [])
37 {
38 $data = [
39 'title' => $entry['title'],
40 'html' => '',
41 'url' => $entry['uri'],
42 'is_archived' => $this->markAsRead,
43 'tags' => '',
44 'created_at' => substr($entry['dateAdded'], 0, 10),
45 ];
46
47 if (array_key_exists('tags', $entry) && $entry['tags'] != '') {
48 $data['tags'] = $entry['tags'];
49 }
50
51 return $data;
52 }
53}
diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
new file mode 100644
index 00000000..cf4c785c
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
@@ -0,0 +1,140 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5use Wallabag\CoreBundle\Entity\Entry;
6
7class InstapaperImport extends AbstractImport
8{
9 private $filepath;
10
11 /**
12 * {@inheritdoc}
13 */
14 public function getName()
15 {
16 return 'Instapaper';
17 }
18
19 /**
20 * {@inheritdoc}
21 */
22 public function getUrl()
23 {
24 return 'import_instapaper';
25 }
26
27 /**
28 * {@inheritdoc}
29 */
30 public function getDescription()
31 {
32 return 'import.instapaper.description';
33 }
34
35 /**
36 * Set file path to the json file.
37 *
38 * @param string $filepath
39 */
40 public function setFilepath($filepath)
41 {
42 $this->filepath = $filepath;
43
44 return $this;
45 }
46
47 /**
48 * {@inheritdoc}
49 */
50 public function import()
51 {
52 if (!$this->user) {
53 $this->logger->error('InstapaperImport: user is not defined');
54
55 return false;
56 }
57
58 if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
59 $this->logger->error('InstapaperImport: unable to read file', ['filepath' => $this->filepath]);
60
61 return false;
62 }
63
64 $entries = [];
65 $handle = fopen($this->filepath, 'r');
66 while (($data = fgetcsv($handle, 10240)) !== false) {
67 if ('URL' === $data[0]) {
68 continue;
69 }
70
71 $entries[] = [
72 'url' => $data[0],
73 'title' => $data[1],
74 'status' => $data[3],
75 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred',
76 'is_starred' => $data[3] === 'Starred',
77 'content_type' => '',
78 'language' => '',
79 ];
80 }
81 fclose($handle);
82
83 if (empty($entries)) {
84 $this->logger->error('InstapaperImport: no entries in imported file');
85
86 return false;
87 }
88
89 if ($this->producer) {
90 $this->parseEntriesForProducer($entries);
91
92 return true;
93 }
94
95 $this->parseEntries($entries);
96
97 return true;
98 }
99
100 /**
101 * {@inheritdoc}
102 */
103 public function parseEntry(array $importedEntry)
104 {
105 $existingEntry = $this->em
106 ->getRepository('WallabagCoreBundle:Entry')
107 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
108
109 if (false !== $existingEntry) {
110 ++$this->skippedEntries;
111
112 return;
113 }
114
115 $entry = new Entry($this->user);
116 $entry->setUrl($importedEntry['url']);
117 $entry->setTitle($importedEntry['title']);
118
119 // update entry with content (in case fetching failed, the given entry will be return)
120 $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry);
121
122 $entry->setArchived($importedEntry['is_archived']);
123 $entry->setStarred($importedEntry['is_starred']);
124
125 $this->em->persist($entry);
126 ++$this->importedEntries;
127
128 return $entry;
129 }
130
131 /**
132 * {@inheritdoc}
133 */
134 protected function setEntryAsRead(array $importedEntry)
135 {
136 $importedEntry['is_archived'] = 1;
137
138 return $importedEntry;
139 }
140}
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 798cfdae..327e2500 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -6,31 +6,34 @@ use Psr\Log\NullLogger;
6use Doctrine\ORM\EntityManager; 6use Doctrine\ORM\EntityManager;
7use GuzzleHttp\Client; 7use GuzzleHttp\Client;
8use GuzzleHttp\Exception\RequestException; 8use GuzzleHttp\Exception\RequestException;
9use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
10use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
11use Wallabag\CoreBundle\Helper\ContentProxy; 10use Wallabag\CoreBundle\Helper\ContentProxy;
12use Craue\ConfigBundle\Util\Config;
13 11
14class PocketImport extends AbstractImport 12class PocketImport extends AbstractImport
15{ 13{
16 private $user;
17 private $client; 14 private $client;
18 private $consumerKey; 15 private $accessToken;
19 private $skippedEntries = 0;
20 private $importedEntries = 0;
21 private $markAsRead;
22 protected $accessToken;
23 16
24 public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) 17 const NB_ELEMENTS = 5000;
18
19 public function __construct(EntityManager $em, ContentProxy $contentProxy)
25 { 20 {
26 $this->user = $tokenStorage->getToken()->getUser();
27 $this->em = $em; 21 $this->em = $em;
28 $this->contentProxy = $contentProxy; 22 $this->contentProxy = $contentProxy;
29 $this->consumerKey = $craueConfig->get('pocket_consumer_key');
30 $this->logger = new NullLogger(); 23 $this->logger = new NullLogger();
31 } 24 }
32 25
33 /** 26 /**
27 * Only used for test purpose.
28 *
29 * @return string
30 */
31 public function getAccessToken()
32 {
33 return $this->accessToken;
34 }
35
36 /**
34 * {@inheritdoc} 37 * {@inheritdoc}
35 */ 38 */
36 public function getName() 39 public function getName()
@@ -66,7 +69,7 @@ class PocketImport extends AbstractImport
66 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', 69 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
67 [ 70 [
68 'body' => json_encode([ 71 'body' => json_encode([
69 'consumer_key' => $this->consumerKey, 72 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
70 'redirect_uri' => $redirectUri, 73 'redirect_uri' => $redirectUri,
71 ]), 74 ]),
72 ] 75 ]
@@ -96,7 +99,7 @@ class PocketImport extends AbstractImport
96 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', 99 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
97 [ 100 [
98 'body' => json_encode([ 101 'body' => json_encode([
99 'consumer_key' => $this->consumerKey, 102 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
100 'code' => $code, 103 'code' => $code,
101 ]), 104 ]),
102 ] 105 ]
@@ -116,38 +119,22 @@ class PocketImport extends AbstractImport
116 } 119 }
117 120
118 /** 121 /**
119 * Set whether articles must be all marked as read.
120 *
121 * @param bool $markAsRead
122 */
123 public function setMarkAsRead($markAsRead)
124 {
125 $this->markAsRead = $markAsRead;
126
127 return $this;
128 }
129
130 /**
131 * Get whether articles must be all marked as read.
132 */
133 public function getMarkAsRead()
134 {
135 return $this->markAsRead;
136 }
137
138 /**
139 * {@inheritdoc} 122 * {@inheritdoc}
140 */ 123 */
141 public function import() 124 public function import($offset = 0)
142 { 125 {
126 static $run = 0;
127
143 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', 128 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
144 [ 129 [
145 'body' => json_encode([ 130 'body' => json_encode([
146 'consumer_key' => $this->consumerKey, 131 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
147 'access_token' => $this->accessToken, 132 'access_token' => $this->accessToken,
148 'detailType' => 'complete', 133 'detailType' => 'complete',
149 'state' => 'all', 134 'state' => 'all',
150 'sort' => 'oldest', 135 'sort' => 'newest',
136 'count' => self::NB_ELEMENTS,
137 'offset' => $offset,
151 ]), 138 ]),
152 ] 139 ]
153 ); 140 );
@@ -162,20 +149,24 @@ class PocketImport extends AbstractImport
162 149
163 $entries = $response->json(); 150 $entries = $response->json();
164 151
165 $this->parseEntries($entries['list']); 152 if ($this->producer) {
153 $this->parseEntriesForProducer($entries['list']);
154 } else {
155 $this->parseEntries($entries['list']);
156 }
166 157
167 return true; 158 // if we retrieve exactly the amount of items requested it means we can get more
168 } 159 // re-call import and offset item by the amount previous received:
160 // - first call get 5k offset 0
161 // - second call get 5k offset 5k
162 // - and so on
163 if (count($entries['list']) === self::NB_ELEMENTS) {
164 ++$run;
169 165
170 /** 166 return $this->import(self::NB_ELEMENTS * $run);
171 * {@inheritdoc} 167 }
172 */ 168
173 public function getSummary() 169 return true;
174 {
175 return [
176 'skipped' => $this->skippedEntries,
177 'imported' => $this->importedEntries,
178 ];
179 } 170 }
180 171
181 /** 172 /**
@@ -189,77 +180,75 @@ class PocketImport extends AbstractImport
189 } 180 }
190 181
191 /** 182 /**
192 * @see https://getpocket.com/developer/docs/v3/retrieve 183 * {@inheritdoc}
193 * 184 *
194 * @param $entries 185 * @see https://getpocket.com/developer/docs/v3/retrieve
195 */ 186 */
196 private function parseEntries($entries) 187 public function parseEntry(array $importedEntry)
197 { 188 {
198 $i = 1; 189 $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
199 190
200 foreach ($entries as $pocketEntry) { 191 $existingEntry = $this->em
201 $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; 192 ->getRepository('WallabagCoreBundle:Entry')
202 193 ->findByUrlAndUserId($url, $this->user->getId());
203 $existingEntry = $this->em 194
204 ->getRepository('WallabagCoreBundle:Entry') 195 if (false !== $existingEntry) {
205 ->findByUrlAndUserId($url, $this->user->getId()); 196 ++$this->skippedEntries;
206 197
207 if (false !== $existingEntry) { 198 return;
208 ++$this->skippedEntries; 199 }
209 continue; 200
210 } 201 $entry = new Entry($this->user);
211 202 $entry->setUrl($url);
212 $entry = new Entry($this->user); 203
213 $entry = $this->fetchContent($entry, $url); 204 // update entry with content (in case fetching failed, the given entry will be return)
214 205 $entry = $this->fetchContent($entry, $url);
215 // jump to next entry in case of problem while getting content 206
216 if (false === $entry) { 207 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
217 ++$this->skippedEntries; 208 $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
218 continue; 209
219 } 210 // 0 or 1 - 1 If the item is starred
220 211 $entry->setStarred($importedEntry['favorite'] == 1);
221 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted 212
222 if ($pocketEntry['status'] == 1 || $this->markAsRead) { 213 $title = 'Untitled';
223 $entry->setArchived(true); 214 if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
224 } 215 $title = $importedEntry['resolved_title'];
225 216 } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
226 // 0 or 1 - 1 If the item is starred 217 $title = $importedEntry['given_title'];
227 if ($pocketEntry['favorite'] == 1) { 218 }
228 $entry->setStarred(true); 219
229 } 220 $entry->setTitle($title);
230 221
231 $title = 'Untitled'; 222 // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
232 if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { 223 if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
233 $title = $pocketEntry['resolved_title']; 224 $entry->setPreviewPicture($importedEntry['images'][1]['src']);
234 } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { 225 }
235 $title = $pocketEntry['given_title']; 226
236 } 227 if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
237 228 $this->contentProxy->assignTagsToEntry(
238 $entry->setTitle($title); 229 $entry,
239 230 array_keys($importedEntry['tags']),
240 // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image 231 $this->em->getUnitOfWork()->getScheduledEntityInsertions()
241 if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { 232 );
242 $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
243 }
244
245 if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
246 $this->contentProxy->assignTagsToEntry(
247 $entry,
248 array_keys($pocketEntry['tags'])
249 );
250 }
251
252 $this->em->persist($entry);
253 ++$this->importedEntries;
254
255 // flush every 20 entries
256 if (($i % 20) === 0) {
257 $this->em->flush();
258 $this->em->clear($entry);
259 }
260 ++$i;
261 } 233 }
262 234
263 $this->em->flush(); 235 if (!empty($importedEntry['time_added'])) {
236 $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
237 }
238
239 $this->em->persist($entry);
240 ++$this->importedEntries;
241
242 return $entry;
243 }
244
245 /**
246 * {@inheritdoc}
247 */
248 protected function setEntryAsRead(array $importedEntry)
249 {
250 $importedEntry['status'] = '1';
251
252 return $importedEntry;
264 } 253 }
265} 254}
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
new file mode 100644
index 00000000..b8c0f777
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -0,0 +1,134 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5use Wallabag\CoreBundle\Entity\Entry;
6
7class ReadabilityImport extends AbstractImport
8{
9 private $filepath;
10
11 /**
12 * {@inheritdoc}
13 */
14 public function getName()
15 {
16 return 'Readability';
17 }
18
19 /**
20 * {@inheritdoc}
21 */
22 public function getUrl()
23 {
24 return 'import_readability';
25 }
26
27 /**
28 * {@inheritdoc}
29 */
30 public function getDescription()
31 {
32 return 'import.readability.description';
33 }
34
35 /**
36 * Set file path to the json file.
37 *
38 * @param string $filepath
39 */
40 public function setFilepath($filepath)
41 {
42 $this->filepath = $filepath;
43
44 return $this;
45 }
46
47 /**
48 * {@inheritdoc}
49 */
50 public function import()
51 {
52 if (!$this->user) {
53 $this->logger->error('ReadabilityImport: user is not defined');
54
55 return false;
56 }
57
58 if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
59 $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]);
60
61 return false;
62 }
63
64 $data = json_decode(file_get_contents($this->filepath), true);
65
66 if (empty($data) || empty($data['bookmarks'])) {
67 $this->logger->error('ReadabilityImport: no entries in imported file');
68
69 return false;
70 }
71
72 if ($this->producer) {
73 $this->parseEntriesForProducer($data['bookmarks']);
74
75 return true;
76 }
77
78 $this->parseEntries($data['bookmarks']);
79
80 return true;
81 }
82
83 /**
84 * {@inheritdoc}
85 */
86 public function parseEntry(array $importedEntry)
87 {
88 $existingEntry = $this->em
89 ->getRepository('WallabagCoreBundle:Entry')
90 ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
91
92 if (false !== $existingEntry) {
93 ++$this->skippedEntries;
94
95 return;
96 }
97
98 $data = [
99 'title' => $importedEntry['article__title'],
100 'url' => $importedEntry['article__url'],
101 'content_type' => '',
102 'language' => '',
103 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
104 'is_starred' => $importedEntry['favorite'],
105 'created_at' => $importedEntry['date_added'],
106 ];
107
108 $entry = new Entry($this->user);
109 $entry->setUrl($data['url']);
110 $entry->setTitle($data['title']);
111
112 // update entry with content (in case fetching failed, the given entry will be return)
113 $entry = $this->fetchContent($entry, $data['url'], $data);
114
115 $entry->setArchived($data['is_archived']);
116 $entry->setStarred($data['is_starred']);
117 $entry->setCreatedAt(new \DateTime($data['created_at']));
118
119 $this->em->persist($entry);
120 ++$this->importedEntries;
121
122 return $entry;
123 }
124
125 /**
126 * {@inheritdoc}
127 */
128 protected function setEntryAsRead(array $importedEntry)
129 {
130 $importedEntry['archive'] = 1;
131
132 return $importedEntry;
133 }
134}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php
index a1cc085b..702da057 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagImport.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php
@@ -3,15 +3,10 @@
3namespace Wallabag\ImportBundle\Import; 3namespace Wallabag\ImportBundle\Import;
4 4
5use Wallabag\CoreBundle\Entity\Entry; 5use Wallabag\CoreBundle\Entity\Entry;
6use Wallabag\UserBundle\Entity\User;
7 6
8abstract class WallabagImport extends AbstractImport 7abstract class WallabagImport extends AbstractImport
9{ 8{
10 protected $user;
11 protected $skippedEntries = 0;
12 protected $importedEntries = 0;
13 protected $filepath; 9 protected $filepath;
14 protected $markAsRead;
15 // untitled in all languages from v1 10 // untitled in all languages from v1
16 protected $untitled = [ 11 protected $untitled = [
17 'Untitled', 12 'Untitled',
@@ -29,19 +24,6 @@ abstract class WallabagImport extends AbstractImport
29 ]; 24 ];
30 25
31 /** 26 /**
32 * We define the user in a custom call because on the import command there is no logged in user.
33 * So we can't retrieve user from the `security.token_storage` service.
34 *
35 * @param User $user
36 */
37 public function setUser(User $user)
38 {
39 $this->user = $user;
40
41 return $this;
42 }
43
44 /**
45 * {@inheritdoc} 27 * {@inheritdoc}
46 */ 28 */
47 abstract public function getName(); 29 abstract public function getName();
@@ -76,26 +58,23 @@ abstract class WallabagImport extends AbstractImport
76 $data = json_decode(file_get_contents($this->filepath), true); 58 $data = json_decode(file_get_contents($this->filepath), true);
77 59
78 if (empty($data)) { 60 if (empty($data)) {
61 $this->logger->error('WallabagImport: no entries in imported file');
62
79 return false; 63 return false;
80 } 64 }
81 65
66 if ($this->producer) {
67 $this->parseEntriesForProducer($data);
68
69 return true;
70 }
71
82 $this->parseEntries($data); 72 $this->parseEntries($data);
83 73
84 return true; 74 return true;
85 } 75 }
86 76
87 /** 77 /**
88 * {@inheritdoc}
89 */
90 public function getSummary()
91 {
92 return [
93 'skipped' => $this->skippedEntries,
94 'imported' => $this->importedEntries,
95 ];
96 }
97
98 /**
99 * Set file path to the json file. 78 * Set file path to the json file.
100 * 79 *
101 * @param string $filepath 80 * @param string $filepath
@@ -108,85 +87,60 @@ abstract class WallabagImport extends AbstractImport
108 } 87 }
109 88
110 /** 89 /**
111 * Set whether articles must be all marked as read. 90 * {@inheritdoc}
112 *
113 * @param bool $markAsRead
114 */ 91 */
115 public function setMarkAsRead($markAsRead) 92 public function parseEntry(array $importedEntry)
116 { 93 {
117 $this->markAsRead = $markAsRead; 94 $existingEntry = $this->em
95 ->getRepository('WallabagCoreBundle:Entry')
96 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
118 97
119 return $this; 98 if (false !== $existingEntry) {
120 } 99 ++$this->skippedEntries;
121 100
122 /** 101 return;
123 * Parse and insert all given entries. 102 }
124 *
125 * @param $entries
126 */
127 protected function parseEntries($entries)
128 {
129 $i = 1;
130 103
131 foreach ($entries as $importedEntry) { 104 $data = $this->prepareEntry($importedEntry);
132 $existingEntry = $this->em
133 ->getRepository('WallabagCoreBundle:Entry')
134 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
135 105
136 if (false !== $existingEntry) { 106 $entry = new Entry($this->user);
137 ++$this->skippedEntries; 107 $entry->setUrl($data['url']);
138 continue; 108 $entry->setTitle($data['title']);
139 }
140 109
141 $data = $this->prepareEntry($importedEntry, $this->markAsRead); 110 // update entry with content (in case fetching failed, the given entry will be return)
111 $entry = $this->fetchContent($entry, $data['url'], $data);
142 112
143 $entry = $this->fetchContent( 113 if (array_key_exists('tags', $data)) {
144 new Entry($this->user), 114 $this->contentProxy->assignTagsToEntry(
145 $importedEntry['url'], 115 $entry,
146 $data 116 $data['tags'],
117 $this->em->getUnitOfWork()->getScheduledEntityInsertions()
147 ); 118 );
119 }
148 120
149 // jump to next entry in case of problem while getting content 121 if (isset($importedEntry['preview_picture'])) {
150 if (false === $entry) { 122 $entry->setPreviewPicture($importedEntry['preview_picture']);
151 ++$this->skippedEntries;
152 continue;
153 }
154
155 if (array_key_exists('tags', $data)) {
156 $this->contentProxy->assignTagsToEntry(
157 $entry,
158 $data['tags']
159 );
160 }
161
162 if (isset($importedEntry['preview_picture'])) {
163 $entry->setPreviewPicture($importedEntry['preview_picture']);
164 }
165
166 $entry->setArchived($data['is_archived']);
167 $entry->setStarred($data['is_starred']);
168
169 $this->em->persist($entry);
170 ++$this->importedEntries;
171
172 // flush every 20 entries
173 if (($i % 20) === 0) {
174 $this->em->flush();
175 $this->em->clear($entry);
176 }
177 ++$i;
178 } 123 }
179 124
180 $this->em->flush(); 125 $entry->setArchived($data['is_archived']);
126 $entry->setStarred($data['is_starred']);
127
128 if (!empty($data['created_at'])) {
129 $entry->setCreatedAt(new \DateTime($data['created_at']));
130 }
131
132 $this->em->persist($entry);
133 ++$this->importedEntries;
134
135 return $entry;
181 } 136 }
182 137
183 /** 138 /**
184 * This should return a cleaned array for a given entry to be given to `updateEntry`. 139 * This should return a cleaned array for a given entry to be given to `updateEntry`.
185 * 140 *
186 * @param array $entry Data from the imported file 141 * @param array $entry Data from the imported file
187 * @param bool $markAsRead Should we mark as read content?
188 * 142 *
189 * @return array 143 * @return array
190 */ 144 */
191 abstract protected function prepareEntry($entry = [], $markAsRead = false); 145 abstract protected function prepareEntry($entry = []);
192} 146}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
index 6cf3467a..4f001062 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
@@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport
31 /** 31 /**
32 * {@inheritdoc} 32 * {@inheritdoc}
33 */ 33 */
34 protected function prepareEntry($entry = [], $markAsRead = false) 34 protected function prepareEntry($entry = [])
35 { 35 {
36 $data = [ 36 $data = [
37 'title' => $entry['title'], 37 'title' => $entry['title'],
@@ -39,9 +39,10 @@ class WallabagV1Import extends WallabagImport
39 'url' => $entry['url'], 39 'url' => $entry['url'],
40 'content_type' => '', 40 'content_type' => '',
41 'language' => '', 41 'language' => '',
42 'is_archived' => $entry['is_read'] || $markAsRead, 42 'is_archived' => $entry['is_read'] || $this->markAsRead,
43 'is_starred' => $entry['is_fav'], 43 'is_starred' => $entry['is_fav'],
44 'tags' => '', 44 'tags' => '',
45 'created_at' => '',
45 ]; 46 ];
46 47
47 // force content to be refreshed in case on bad fetch in the v1 installation 48 // force content to be refreshed in case on bad fetch in the v1 installation
@@ -56,4 +57,14 @@ class WallabagV1Import extends WallabagImport
56 57
57 return $data; 58 return $data;
58 } 59 }
60
61 /**
62 * {@inheritdoc}
63 */
64 protected function setEntryAsRead(array $importedEntry)
65 {
66 $importedEntry['is_read'] = 1;
67
68 return $importedEntry;
69 }
59} 70}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
index d0035b63..37c8ca14 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
@@ -31,12 +31,22 @@ class WallabagV2Import extends WallabagImport
31 /** 31 /**
32 * {@inheritdoc} 32 * {@inheritdoc}
33 */ 33 */
34 protected function prepareEntry($entry = [], $markAsRead = false) 34 protected function prepareEntry($entry = [])
35 { 35 {
36 return [ 36 return [
37 'html' => $entry['content'], 37 'html' => $entry['content'],
38 'content_type' => $entry['mimetype'], 38 'content_type' => $entry['mimetype'],
39 'is_archived' => ($entry['is_archived'] || $markAsRead), 39 'is_archived' => ($entry['is_archived'] || $this->markAsRead),
40 ] + $entry; 40 ] + $entry;
41 } 41 }
42
43 /**
44 * {@inheritdoc}
45 */
46 protected function setEntryAsRead(array $importedEntry)
47 {
48 $importedEntry['is_archived'] = 1;
49
50 return $importedEntry;
51 }
42} 52}
diff --git a/src/Wallabag/ImportBundle/Redis/Producer.php b/src/Wallabag/ImportBundle/Redis/Producer.php
new file mode 100644
index 00000000..fedc3e57
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Redis/Producer.php
@@ -0,0 +1,36 @@
1<?php
2
3namespace Wallabag\ImportBundle\Redis;
4
5use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
6use Simpleue\Queue\RedisQueue;
7
8/**
9 * This is a proxy class for "Simpleue\Queue\RedisQueue".
10 * It allow us to use the same way to publish a message between RabbitMQ & Redis: publish().
11 *
12 * It implements the ProducerInterface of RabbitMQ (yes it's ugly) so we can have the same
13 * kind of class which implements the same interface.
14 * So we can inject either a RabbitMQ producer or a Redis producer with the same signature
15 */
16class Producer implements ProducerInterface
17{
18 private $queue;
19
20 public function __construct(RedisQueue $queue)
21 {
22 $this->queue = $queue;
23 }
24
25 /**
26 * Publish a message in the Redis queue.
27 *
28 * @param string $msgBody
29 * @param string $routingKey NOT USED
30 * @param array $additionalProperties NOT USED
31 */
32 public function publish($msgBody, $routingKey = '', $additionalProperties = array())
33 {
34 $this->queue->sendJob($msgBody);
35 }
36}
diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml
new file mode 100644
index 00000000..70b8a0d4
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml
@@ -0,0 +1,51 @@
1# RabbitMQ stuff
2services:
3 wallabag_import.consumer.amqp.pocket:
4 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
5 arguments:
6 - "@doctrine.orm.entity_manager"
7 - "@wallabag_user.user_repository"
8 - "@wallabag_import.pocket.import"
9 - "@logger"
10 wallabag_import.consumer.amqp.readability:
11 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
12 arguments:
13 - "@doctrine.orm.entity_manager"
14 - "@wallabag_user.user_repository"
15 - "@wallabag_import.readability.import"
16 - "@logger"
17 wallabag_import.consumer.amqp.instapaper:
18 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
19 arguments:
20 - "@doctrine.orm.entity_manager"
21 - "@wallabag_user.user_repository"
22 - "@wallabag_import.instapaper.import"
23 - "@logger"
24 wallabag_import.consumer.amqp.wallabag_v1:
25 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
26 arguments:
27 - "@doctrine.orm.entity_manager"
28 - "@wallabag_user.user_repository"
29 - "@wallabag_import.wallabag_v1.import"
30 - "@logger"
31 wallabag_import.consumer.amqp.wallabag_v2:
32 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
33 arguments:
34 - "@doctrine.orm.entity_manager"
35 - "@wallabag_user.user_repository"
36 - "@wallabag_import.wallabag_v2.import"
37 - "@logger"
38 wallabag_import.consumer.amqp.firefox:
39 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
40 arguments:
41 - "@doctrine.orm.entity_manager"
42 - "@wallabag_user.user_repository"
43 - "@wallabag_import.firefox.import"
44 - "@logger"
45 wallabag_import.consumer.amqp.chrome:
46 class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
47 arguments:
48 - "@doctrine.orm.entity_manager"
49 - "@wallabag_user.user_repository"
50 - "@wallabag_import.chrome.import"
51 - "@logger"
diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml
new file mode 100644
index 00000000..0a81e1b5
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml
@@ -0,0 +1,141 @@
1# Redis stuff
2services:
3 # readability
4 wallabag_import.queue.redis.readability:
5 class: Simpleue\Queue\RedisQueue
6 arguments:
7 - "@wallabag_core.redis.client"
8 - "wallabag.import.readability"
9
10 wallabag_import.producer.redis.readability:
11 class: Wallabag\ImportBundle\Redis\Producer
12 arguments:
13 - "@wallabag_import.queue.redis.readability"
14
15 wallabag_import.consumer.redis.readability:
16 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
17 arguments:
18 - "@doctrine.orm.entity_manager"
19 - "@wallabag_user.user_repository"
20 - "@wallabag_import.readability.import"
21 - "@logger"
22
23 # instapaper
24 wallabag_import.queue.redis.instapaper:
25 class: Simpleue\Queue\RedisQueue
26 arguments:
27 - "@wallabag_core.redis.client"
28 - "wallabag.import.instapaper"
29
30 wallabag_import.producer.redis.instapaper:
31 class: Wallabag\ImportBundle\Redis\Producer
32 arguments:
33 - "@wallabag_import.queue.redis.instapaper"
34
35 wallabag_import.consumer.redis.instapaper:
36 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
37 arguments:
38 - "@doctrine.orm.entity_manager"
39 - "@wallabag_user.user_repository"
40 - "@wallabag_import.instapaper.import"
41 - "@logger"
42
43 # pocket
44 wallabag_import.queue.redis.pocket:
45 class: Simpleue\Queue\RedisQueue
46 arguments:
47 - "@wallabag_core.redis.client"
48 - "wallabag.import.pocket"
49
50 wallabag_import.producer.redis.pocket:
51 class: Wallabag\ImportBundle\Redis\Producer
52 arguments:
53 - "@wallabag_import.queue.redis.pocket"
54
55 wallabag_import.consumer.redis.pocket:
56 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
57 arguments:
58 - "@doctrine.orm.entity_manager"
59 - "@wallabag_user.user_repository"
60 - "@wallabag_import.pocket.import"
61 - "@logger"
62
63 # wallabag v1
64 wallabag_import.queue.redis.wallabag_v1:
65 class: Simpleue\Queue\RedisQueue
66 arguments:
67 - "@wallabag_core.redis.client"
68 - "wallabag.import.wallabag_v1"
69
70 wallabag_import.producer.redis.wallabag_v1:
71 class: Wallabag\ImportBundle\Redis\Producer
72 arguments:
73 - "@wallabag_import.queue.redis.wallabag_v1"
74
75 wallabag_import.consumer.redis.wallabag_v1:
76 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
77 arguments:
78 - "@doctrine.orm.entity_manager"
79 - "@wallabag_user.user_repository"
80 - "@wallabag_import.wallabag_v1.import"
81 - "@logger"
82
83 # wallabag v2
84 wallabag_import.queue.redis.wallabag_v2:
85 class: Simpleue\Queue\RedisQueue
86 arguments:
87 - "@wallabag_core.redis.client"
88 - "wallabag.import.wallabag_v2"
89
90 wallabag_import.producer.redis.wallabag_v2:
91 class: Wallabag\ImportBundle\Redis\Producer
92 arguments:
93 - "@wallabag_import.queue.redis.wallabag_v2"
94
95 wallabag_import.consumer.redis.wallabag_v2:
96 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
97 arguments:
98 - "@doctrine.orm.entity_manager"
99 - "@wallabag_user.user_repository"
100 - "@wallabag_import.wallabag_v2.import"
101 - "@logger"
102
103 # firefox
104 wallabag_import.queue.redis.firefox:
105 class: Simpleue\Queue\RedisQueue
106 arguments:
107 - "@wallabag_core.redis.client"
108 - "wallabag.import.firefox"
109
110 wallabag_import.producer.redis.firefox:
111 class: Wallabag\ImportBundle\Redis\Producer
112 arguments:
113 - "@wallabag_import.queue.redis.firefox"
114
115 wallabag_import.consumer.redis.firefox:
116 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
117 arguments:
118 - "@doctrine.orm.entity_manager"
119 - "@wallabag_user.user_repository"
120 - "@wallabag_import.firefox.import"
121 - "@logger"
122
123 # chrome
124 wallabag_import.queue.redis.chrome:
125 class: Simpleue\Queue\RedisQueue
126 arguments:
127 - "@wallabag_core.redis.client"
128 - "wallabag.import.chrome"
129
130 wallabag_import.producer.redis.chrome:
131 class: Wallabag\ImportBundle\Redis\Producer
132 arguments:
133 - "@wallabag_import.queue.redis.chrome"
134
135 wallabag_import.consumer.redis.chrome:
136 class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
137 arguments:
138 - "@doctrine.orm.entity_manager"
139 - "@wallabag_user.user_repository"
140 - "@wallabag_import.chrome.import"
141 - "@logger"
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index 86b44cb3..89adc71b 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -1,3 +1,7 @@
1imports:
2 - { resource: rabbit.yml }
3 - { resource: redis.yml }
4
1services: 5services:
2 wallabag_import.chain: 6 wallabag_import.chain:
3 class: Wallabag\ImportBundle\Import\ImportChain 7 class: Wallabag\ImportBundle\Import\ImportChain
@@ -14,7 +18,6 @@ services:
14 wallabag_import.pocket.import: 18 wallabag_import.pocket.import:
15 class: Wallabag\ImportBundle\Import\PocketImport 19 class: Wallabag\ImportBundle\Import\PocketImport
16 arguments: 20 arguments:
17 - "@security.token_storage"
18 - "@doctrine.orm.entity_manager" 21 - "@doctrine.orm.entity_manager"
19 - "@wallabag_core.content_proxy" 22 - "@wallabag_core.content_proxy"
20 - "@craue_config" 23 - "@craue_config"
@@ -43,3 +46,42 @@ services:
43 - [ setLogger, [ "@logger" ]] 46 - [ setLogger, [ "@logger" ]]
44 tags: 47 tags:
45 - { name: wallabag_import.import, alias: wallabag_v2 } 48 - { name: wallabag_import.import, alias: wallabag_v2 }
49
50 wallabag_import.readability.import:
51 class: Wallabag\ImportBundle\Import\ReadabilityImport
52 arguments:
53 - "@doctrine.orm.entity_manager"
54 - "@wallabag_core.content_proxy"
55 calls:
56 - [ setLogger, [ "@logger" ]]
57 tags:
58 - { name: wallabag_import.import, alias: readability }
59
60 wallabag_import.instapaper.import:
61 class: Wallabag\ImportBundle\Import\InstapaperImport
62 arguments:
63 - "@doctrine.orm.entity_manager"
64 - "@wallabag_core.content_proxy"
65 calls:
66 - [ setLogger, [ "@logger" ]]
67 tags:
68 - { name: wallabag_import.import, alias: instapaper }
69
70 wallabag_import.firefox.import:
71 class: Wallabag\ImportBundle\Import\FirefoxImport
72 arguments:
73 - "@doctrine.orm.entity_manager"
74 - "@wallabag_core.content_proxy"
75 calls:
76 - [ setLogger, [ "@logger" ]]
77 tags:
78 - { name: wallabag_import.import, alias: firefox }
79 wallabag_import.chrome.import:
80 class: Wallabag\ImportBundle\Import\ChromeImport
81 arguments:
82 - "@doctrine.orm.entity_manager"
83 - "@wallabag_core.content_proxy"
84 calls:
85 - [ setLogger, [ "@logger" ]]
86 tags:
87 - { name: wallabag_import.import, alias: chrome }
diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig
new file mode 100644
index 00000000..ead828c6
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig
@@ -0,0 +1,43 @@
1{% extends "WallabagCoreBundle::layout.html.twig" %}
2
3{% block title %}{{ 'import.chrome.page_title'|trans }}{% endblock %}
4
5{% block content %}
6<div class="row">
7 <div class="col s12">
8 <div class="card-panel settings">
9 <div class="row">
10 <blockquote>{{ import.description|trans|raw }}</blockquote>
11 <p>{{ 'import.chrome.how_to'|trans }}</p>
12
13 <div class="col s12">
14 {{ form_start(form, {'method': 'POST'}) }}
15 {{ form_errors(form) }}
16 <div class="row">
17 <div class="file-field input-field col s12">
18 {{ form_errors(form.file) }}
19 <div class="btn">
20 <span>{{ form.file.vars.label|trans }}</span>
21 {{ form_widget(form.file) }}
22 </div>
23 <div class="file-path-wrapper">
24 <input class="file-path validate" type="text">
25 </div>
26 </div>
27 <div class="input-field col s6 with-checkbox">
28 <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
29 {{ form_widget(form.mark_as_read) }}
30 {{ form_label(form.mark_as_read) }}
31 </div>
32 </div>
33
34 {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
35
36 {{ form_rest(form) }}
37 </form>
38 </div>
39 </div>
40 </div>
41 </div>
42</div>
43{% endblock %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig
new file mode 100644
index 00000000..f975da3f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig
@@ -0,0 +1,43 @@
1{% extends "WallabagCoreBundle::layout.html.twig" %}
2
3{% block title %}{{ 'import.firefox.page_title'|trans }}{% endblock %}
4
5{% block content %}
6<div class="row">
7 <div class="col s12">
8 <div class="card-panel settings">
9 <div class="row">
10 <blockquote>{{ import.description|trans|raw }}</blockquote>
11 <p>{{ 'import.firefox.how_to'|trans }}</p>
12
13 <div class="col s12">
14 {{ form_start(form, {'method': 'POST'}) }}
15 {{ form_errors(form) }}
16 <div class="row">
17 <div class="file-field input-field col s12">
18 {{ form_errors(form.file) }}
19 <div class="btn">
20 <span>{{ form.file.vars.label|trans }}</span>
21 {{ form_widget(form.file) }}
22 </div>
23 <div class="file-path-wrapper">
24 <input class="file-path validate" type="text">
25 </div>
26 </div>
27 <div class="input-field col s6 with-checkbox">
28 <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
29 {{ form_widget(form.mark_as_read) }}
30 {{ form_label(form.mark_as_read) }}
31 </div>
32 </div>
33
34 {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
35
36 {{ form_rest(form) }}
37 </form>
38 </div>
39 </div>
40 </div>
41 </div>
42</div>
43{% endblock %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig
new file mode 100644
index 00000000..2390a41f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig
@@ -0,0 +1,8 @@
1{% set redis = craue_setting('import_with_redis') %}
2{% set rabbit = craue_setting('import_with_rabbitmq') %}
3
4{% if redis or rabbit %}
5 <div class="card-panel yellow darken-1 black-text">
6 {{ 'import.worker.enabled'|trans }} <strong>{% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}</strong>
7 </div>
8{% endif %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig
new file mode 100644
index 00000000..a2633698
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig
@@ -0,0 +1,23 @@
1{% if nbRedisMessages is defined and nbRedisMessages > 0 %}
2 <script>
3 Materialize.toast('Messages in queue: {{ nbRedisMessages }}', 4000);
4 </script>
5{% endif %}
6
7{% if nbRabbitMessages is defined and nbRabbitMessages > 0 %}
8 <script>
9 Materialize.toast('Messages in queue: {{ nbRabbitMessages }}', 4000);
10 </script>
11{% endif %}
12
13{% if redisNotInstalled is defined and redisNotInstalled %}
14 <div class="card-panel red darken-1 white-text">
15 {{ 'flashes.import.error.redis_enabled_not_installed'|trans|raw }}
16 </div>
17{% endif %}
18
19{% if rabbitNotInstalled is defined and rabbitNotInstalled %}
20 <div class="card-panel red darken-1 white-text">
21 {{ 'flashes.import.error.rabbit_enabled_not_installed'|trans|raw }}
22 </div>
23{% endif %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig
index aebbfa20..6ea5e0f4 100644
--- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig
+++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig
@@ -11,7 +11,7 @@
11 {% for import in imports %} 11 {% for import in imports %}
12 <li> 12 <li>
13 <h5>{{ import.name }}</h5> 13 <h5>{{ import.name }}</h5>
14 <blockquote>{{ import.description|trans }}</blockquote> 14 <blockquote>{{ import.description|trans|raw }}</blockquote>
15 <p><a class="waves-effect waves-light btn" href="{{ path(import.url) }}">{{ 'import.action.import_contents'|trans }}</a></p> 15 <p><a class="waves-effect waves-light btn" href="{{ path(import.url) }}">{{ 'import.action.import_contents'|trans }}</a></p>
16 </li> 16 </li>
17 {% endfor %} 17 {% endfor %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig
new file mode 100644
index 00000000..5789361f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig
@@ -0,0 +1,45 @@
1{% extends "WallabagCoreBundle::layout.html.twig" %}
2
3{% block title %}{{ 'import.instapaper.page_title'|trans }}{% endblock %}
4
5{% block content %}
6<div class="row">
7 <div class="col s12">
8 <div class="card-panel settings">
9 {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
10
11 <div class="row">
12 <blockquote>{{ import.description|trans }}</blockquote>
13 <p>{{ 'import.instapaper.how_to'|trans }}</p>
14
15 <div class="col s12">
16 {{ form_start(form, {'method': 'POST'}) }}
17 {{ form_errors(form) }}
18 <div class="row">
19 <div class="file-field input-field col s12">
20 {{ form_errors(form.file) }}
21 <div class="btn">
22 <span>{{ form.file.vars.label|trans }}</span>
23 {{ form_widget(form.file) }}
24 </div>
25 <div class="file-path-wrapper">
26 <input class="file-path validate" type="text">
27 </div>
28 </div>
29 <div class="input-field col s6 with-checkbox">
30 <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
31 {{ form_widget(form.mark_as_read) }}
32 {{ form_label(form.mark_as_read) }}
33 </div>
34 </div>
35
36 {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
37
38 {{ form_rest(form) }}
39 </form>
40 </div>
41 </div>
42 </div>
43 </div>
44</div>
45{% endblock %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
index 401dbd33..6195fa07 100644
--- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
+++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
@@ -6,15 +6,13 @@
6<div class="row"> 6<div class="row">
7 <div class="col s12"> 7 <div class="col s12">
8 <div class="card-panel settings"> 8 <div class="card-panel settings">
9 {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
10
9 {% if not has_consumer_key %} 11 {% if not has_consumer_key %}
10 <div class="card-panel red darken-1"> 12 <div class="card-panel red white-text">
11 {{ 'import.pocket.config_missing.description'|trans }} 13 {{ 'import.pocket.config_missing.description'|trans }}
12 14
13 {% if is_granted('ROLE_SUPER_ADMIN') %} 15 {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('config') ~ '">', '%keyurle%':'</a>'})|raw }}
14 {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('craue_config_settings_modify') ~ '#set-import">', '%keyurle%':'</a>'})|raw }}
15 {% else %}
16 {{ 'import.pocket.config_missing.user_message'|trans }}
17 {% endif %}
18 </div> 16 </div>
19 {% endif %} 17 {% endif %}
20 18
@@ -29,7 +27,7 @@
29 {{ form_label(form.mark_as_read) }} 27 {{ form_label(form.mark_as_read) }}
30 </div> 28 </div>
31 </div> 29 </div>
32 <button class="btn waves-effect waves-light" type="submit" name="action"> 30 <button class="btn waves-effect waves-light" type="submit" name="action" {% if not has_consumer_key %}disabled="disabled"{% endif %}>
33 {{ 'import.pocket.connect_to_pocket'|trans }} 31 {{ 'import.pocket.connect_to_pocket'|trans }}
34 </button> 32 </button>
35 </form> 33 </form>
diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
new file mode 100644
index 00000000..74653b0f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
@@ -0,0 +1,45 @@
1{% extends "WallabagCoreBundle::layout.html.twig" %}
2
3{% block title %}{{ 'import.readability.page_title'|trans }}{% endblock %}
4
5{% block content %}
6<div class="row">
7 <div class="col s12">
8 <div class="card-panel settings">
9 {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
10
11 <div class="row">
12 <blockquote>{{ import.description|trans }}</blockquote>
13 <p>{{ 'import.readability.how_to'|trans }}</p>
14
15 <div class="col s12">
16 {{ form_start(form, {'method': 'POST'}) }}
17 {{ form_errors(form) }}
18 <div class="row">
19 <div class="file-field input-field col s12">
20 {{ form_errors(form.file) }}
21 <div class="btn">
22 <span>{{ form.file.vars.label|trans }}</span>
23 {{ form_widget(form.file) }}
24 </div>
25 <div class="file-path-wrapper">
26 <input class="file-path validate" type="text">
27 </div>
28 </div>
29 <div class="input-field col s6 with-checkbox">
30 <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
31 {{ form_widget(form.mark_as_read) }}
32 {{ form_label(form.mark_as_read) }}
33 </div>
34 </div>
35
36 {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
37
38 {{ form_rest(form) }}
39 </form>
40 </div>
41 </div>
42 </div>
43 </div>
44</div>
45{% endblock %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
index 13e24c8c..0b19bc34 100644
--- a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
+++ b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
@@ -6,6 +6,8 @@
6<div class="row"> 6<div class="row">
7 <div class="col s12"> 7 <div class="col s12">
8 <div class="card-panel settings"> 8 <div class="card-panel settings">
9 {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
10
9 <div class="row"> 11 <div class="row">
10 <blockquote>{{ import.description|trans }}</blockquote> 12 <blockquote>{{ import.description|trans }}</blockquote>
11 <p>{{ 'import.wallabag_v1.how_to'|trans }}</p> 13 <p>{{ 'import.wallabag_v1.how_to'|trans }}</p>