From 77a7752a592af9ac821621a34d9955533baf40a0 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 30 Dec 2015 10:06:45 +0100 Subject: Update after previous merge PR #1443 was merged into this branch to handle all import type in the same place. --- app/config/config.yml | 5 +- src/Wallabag/CoreBundle/Command/ImportCommand.php | 124 --------------------- .../CoreBundle/Controller/ImportController.php | 64 ----------- .../DependencyInjection/Configuration.php | 13 --- .../DependencyInjection/WallabagCoreExtension.php | 1 - .../CoreBundle/Form/Type/UploadImportType.php | 29 ----- .../views/themes/material/Import/index.html.twig | 30 ----- .../ImportBundle/Command/ImportCommand.php | 124 +++++++++++++++++++++ .../ImportBundle/Controller/ImportController.php | 48 +++++++- .../DependencyInjection/Configuration.php | 8 ++ .../WallabagImportExtension.php | 1 + .../ImportBundle/Form/Type/UploadImportType.php | 29 +++++ .../Resources/views/Import/index.html.twig | 26 +++++ 13 files changed, 237 insertions(+), 265 deletions(-) delete mode 100644 src/Wallabag/CoreBundle/Command/ImportCommand.php delete mode 100644 src/Wallabag/CoreBundle/Controller/ImportController.php delete mode 100644 src/Wallabag/CoreBundle/Form/Type/UploadImportType.php delete mode 100644 src/Wallabag/CoreBundle/Resources/views/themes/material/Import/index.html.twig create mode 100644 src/Wallabag/ImportBundle/Command/ImportCommand.php create mode 100644 src/Wallabag/ImportBundle/Form/Type/UploadImportType.php diff --git a/app/config/config.yml b/app/config/config.yml index 421b2db5..4d04d002 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -30,8 +30,9 @@ wallabag_core: en: 'English' fr: 'Français' de: 'Deutsch' - import: - allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain'] + +wallabag_import: + allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain'] # Twig Configuration twig: diff --git a/src/Wallabag/CoreBundle/Command/ImportCommand.php b/src/Wallabag/CoreBundle/Command/ImportCommand.php deleted file mode 100644 index 6be6f5e1..00000000 --- a/src/Wallabag/CoreBundle/Command/ImportCommand.php +++ /dev/null @@ -1,124 +0,0 @@ -setName('wallabag:import') - ->setDescription('Import entries from JSON file') - ->addArgument( - 'userId', - InputArgument::REQUIRED, - 'user ID to populate' - ); - } - - protected function execute(InputInterface $input, OutputInterface $output) - { - $now = new \DateTime(); - $output->writeln('Start : '.$now->format('d-m-Y G:i:s').' ---'); - - // Importing CSV on DB via Doctrine ORM - $this->import($input, $output); - - $now = new \DateTime(); - $output->writeln('End : '.$now->format('d-m-Y G:i:s').' ---'); - } - - protected function import(InputInterface $input, OutputInterface $output) - { - // Getting php array of data from CSV - $data = $this->get($input, $output); - - $em = $this->getContainer()->get('doctrine')->getManager(); - // Turning off doctrine default logs queries for saving memory - $em->getConnection()->getConfiguration()->setSQLLogger(null); - - // Define the size of record, the frequency for persisting the data and the current index of records - $size = count($data); - $batchSize = 20; - $i = 1; - - $user = $em->getRepository('WallabagUserBundle:User') - ->findOneById($input->getArgument('userId')); - - if (!is_object($user)) { - throw new Exception('User not found'); - } - - $progress = new ProgressBar($output, $size); - $progress->start(); - - foreach ($data as $object) { - $array = (array) $object; - $entry = $em->getRepository('WallabagCoreBundle:Entry') - ->findOneByUrl($array['url']); - - if (!is_object($entry)) { - $entry = new Entry($user); - $entry->setUrl($array['url']); - } - - $entry->setTitle($array['title']); - $entry->setArchived($array['is_read']); - $entry->setStarred($array['is_fav']); - $entry->setContent($array['content']); - $entry->setReadingTime(Utils::getReadingTime($array['content'])); - - $em->persist($entry); - - if (($i % $batchSize) === 0) { - $em->flush(); - $progress->advance($batchSize); - - $now = new \DateTime(); - $output->writeln(' of entries imported ... | '.$now->format('d-m-Y G:i:s')); - } - ++$i; - } - - $em->flush(); - $em->clear(); - $progress->finish(); - } - - protected function convert($filename) - { - if (!file_exists($filename) || !is_readable($filename)) { - return false; - } - - $header = null; - $data = array(); - - if (($handle = fopen($filename, 'r')) !== false) { - while (($row = fgets($handle)) !== false) { - $data = json_decode($row); - } - fclose($handle); - } - - return $data; - } - - protected function get(InputInterface $input, OutputInterface $output) - { - $filename = __DIR__.'/../../../../web/uploads/import/'.$input->getArgument('userId').'.json'; - - $data = $this->convert($filename); - - return $data; - } -} diff --git a/src/Wallabag/CoreBundle/Controller/ImportController.php b/src/Wallabag/CoreBundle/Controller/ImportController.php deleted file mode 100644 index 33087710..00000000 --- a/src/Wallabag/CoreBundle/Controller/ImportController.php +++ /dev/null @@ -1,64 +0,0 @@ -createForm(new UploadImportType()); - $importForm->handleRequest($request); - $user = $this->getUser(); - $importConfig = $this->container->getParameter('wallabag_core.import'); - - if ($importForm->isValid()) { - $file = $importForm->get('file')->getData(); - $name = $user->getId().'.json'; - $dir = __DIR__.'/../../../../web/uploads/import'; - - if (in_array($file->getMimeType(), $importConfig['allow_mimetypes']) && $file->move($dir, $name)) { - $command = new ImportCommand(); - $command->setContainer($this->container); - $input = new ArrayInput(array('userId' => $user->getId())); - $return = $command->run($input, new NullOutput()); - - if ($return == 0) { - $this->get('session')->getFlashBag()->add( - 'notice', - 'Import successful' - ); - } else { - $this->get('session')->getFlashBag()->add( - 'notice', - 'Import failed' - ); - } - - return $this->redirect('/'); - } else { - $this->get('session')->getFlashBag()->add( - 'notice', - 'Error while processing import. Please verify your import file.' - ); - } - } - - return $this->render('WallabagCoreBundle:Import:index.html.twig', array( - 'form' => array( - 'import' => $importForm->createView(), ), - )); - } -} diff --git a/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php b/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php index fb1941b8..32acd1f1 100644 --- a/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php +++ b/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php @@ -4,7 +4,6 @@ namespace Wallabag\CoreBundle\DependencyInjection; use Symfony\Component\Config\Definition\Builder\TreeBuilder; use Symfony\Component\Config\Definition\ConfigurationInterface; -use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition; class Configuration implements ConfigurationInterface { @@ -18,21 +17,9 @@ class Configuration implements ConfigurationInterface ->arrayNode('languages') ->prototype('scalar')->end() ->end() - ->arrayNode('import') - ->append($this->getAllowMimetypes()) - ->end() ->end() ; return $treeBuilder; } - - private function getAllowMimetypes() - { - $node = new ArrayNodeDefinition('allow_mimetypes'); - - $node->prototype('scalar')->end(); - - return $node; - } } diff --git a/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php b/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php index e2a3ad55..330cc957 100644 --- a/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php +++ b/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php @@ -14,7 +14,6 @@ class WallabagCoreExtension extends Extension $configuration = new Configuration(); $config = $this->processConfiguration($configuration, $configs); $container->setParameter('wallabag_core.languages', $config['languages']); - $container->setParameter('wallabag_core.import', $config['import']); $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load('services.yml'); diff --git a/src/Wallabag/CoreBundle/Form/Type/UploadImportType.php b/src/Wallabag/CoreBundle/Form/Type/UploadImportType.php deleted file mode 100644 index b9a9c465..00000000 --- a/src/Wallabag/CoreBundle/Form/Type/UploadImportType.php +++ /dev/null @@ -1,29 +0,0 @@ -add('file', 'file') - ->add('save', 'submit') - ; - } - - public function getDefaultOptions(array $options) - { - return array( - 'csrf_protection' => false, - ); - } - - public function getName() - { - return 'upload_import_file'; - } -} diff --git a/src/Wallabag/CoreBundle/Resources/views/themes/material/Import/index.html.twig b/src/Wallabag/CoreBundle/Resources/views/themes/material/Import/index.html.twig deleted file mode 100644 index 47004144..00000000 --- a/src/Wallabag/CoreBundle/Resources/views/themes/material/Import/index.html.twig +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "WallabagCoreBundle::layout.html.twig" %} - -{% block title %}{% trans %}import{% endtrans %}{% endblock %} - -{% block content %} -
-
-
-
-
-
- {{ form_errors(form.import) }} -
-
-

{% trans %}Please select your wallabag export and click on the below button to upload and import it.{% endtrans %}

- {{ form_errors(form.import.file) }} - {{ form_widget(form.import.file) }} -
-
- - -
-
-
-
-
-
-{% endblock %} diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php new file mode 100644 index 00000000..3fb8927d --- /dev/null +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -0,0 +1,124 @@ +setName('wallabag:import') + ->setDescription('Import entries from JSON file') + ->addArgument( + 'userId', + InputArgument::REQUIRED, + 'user ID to populate' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $now = new \DateTime(); + $output->writeln('Start : '.$now->format('d-m-Y G:i:s').' ---'); + + // Importing CSV on DB via Doctrine ORM + $this->import($input, $output); + + $now = new \DateTime(); + $output->writeln('End : '.$now->format('d-m-Y G:i:s').' ---'); + } + + protected function import(InputInterface $input, OutputInterface $output) + { + // Getting php array of data from CSV + $data = $this->get($input, $output); + + $em = $this->getContainer()->get('doctrine')->getManager(); + // Turning off doctrine default logs queries for saving memory + $em->getConnection()->getConfiguration()->setSQLLogger(null); + + // Define the size of record, the frequency for persisting the data and the current index of records + $size = count($data); + $batchSize = 20; + $i = 1; + + $user = $em->getRepository('WallabagUserBundle:User') + ->findOneById($input->getArgument('userId')); + + if (!is_object($user)) { + throw new Exception('User not found'); + } + + $progress = new ProgressBar($output, $size); + $progress->start(); + + foreach ($data as $object) { + $array = (array) $object; + $entry = $em->getRepository('WallabagCoreBundle:Entry') + ->findOneByUrl($array['url']); + + if (!is_object($entry)) { + $entry = new Entry($user); + $entry->setUrl($array['url']); + } + + $entry->setTitle($array['title']); + $entry->setArchived($array['is_read']); + $entry->setStarred($array['is_fav']); + $entry->setContent($array['content']); + $entry->setReadingTime(Utils::getReadingTime($array['content'])); + + $em->persist($entry); + + if (($i % $batchSize) === 0) { + $em->flush(); + $progress->advance($batchSize); + + $now = new \DateTime(); + $output->writeln(' of entries imported ... | '.$now->format('d-m-Y G:i:s')); + } + ++$i; + } + + $em->flush(); + $em->clear(); + $progress->finish(); + } + + protected function convert($filename) + { + if (!file_exists($filename) || !is_readable($filename)) { + return false; + } + + $header = null; + $data = array(); + + if (($handle = fopen($filename, 'r')) !== false) { + while (($row = fgets($handle)) !== false) { + $data = json_decode($row); + } + fclose($handle); + } + + return $data; + } + + protected function get(InputInterface $input, OutputInterface $output) + { + $filename = __DIR__.'/../../../../web/uploads/import/'.$input->getArgument('userId').'.json'; + + $data = $this->convert($filename); + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 3569793b..6ebd6a0a 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -4,14 +4,58 @@ namespace Wallabag\ImportBundle\Controller; use Symfony\Bundle\FrameworkBundle\Controller\Controller; use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route; +use Symfony\Component\Console\Input\ArrayInput; +use Symfony\Component\Console\Output\NullOutput; +use Symfony\Component\HttpFoundation\Request; +use Wallabag\ImportBundle\Command\ImportCommand; +use Wallabag\ImportBundle\Form\Type\UploadImportType; class ImportController extends Controller { /** * @Route("/import", name="import") */ - public function importAction() + public function importAction(Request $request) { - return $this->render('WallabagImportBundle:Import:index.html.twig', array()); + $importForm = $this->createForm(new UploadImportType()); + $importForm->handleRequest($request); + $user = $this->getUser(); + + if ($importForm->isValid()) { + $file = $importForm->get('file')->getData(); + $name = $user->getId().'.json'; + $dir = __DIR__.'/../../../../web/uploads/import'; + + if (in_array($file->getMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($dir, $name)) { + $command = new ImportCommand(); + $command->setContainer($this->container); + $input = new ArrayInput(array('userId' => $user->getId())); + $return = $command->run($input, new NullOutput()); + + if ($return == 0) { + $this->get('session')->getFlashBag()->add( + 'notice', + 'Import successful' + ); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'Import failed' + ); + } + + return $this->redirect('/'); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'Error while processing import. Please verify your import file.' + ); + } + } + + return $this->render('WallabagImportBundle:Import:index.html.twig', array( + 'form' => array( + 'import' => $importForm->createView(), ), + )); } } diff --git a/src/Wallabag/ImportBundle/DependencyInjection/Configuration.php b/src/Wallabag/ImportBundle/DependencyInjection/Configuration.php index bacaff31..2ef35463 100644 --- a/src/Wallabag/ImportBundle/DependencyInjection/Configuration.php +++ b/src/Wallabag/ImportBundle/DependencyInjection/Configuration.php @@ -12,6 +12,14 @@ class Configuration implements ConfigurationInterface $treeBuilder = new TreeBuilder(); $rootNode = $treeBuilder->root('wallabag_import'); + $rootNode + ->children() + ->arrayNode('allow_mimetypes') + ->prototype('scalar')->end() + ->end() + ->end() + ; + return $treeBuilder; } } diff --git a/src/Wallabag/ImportBundle/DependencyInjection/WallabagImportExtension.php b/src/Wallabag/ImportBundle/DependencyInjection/WallabagImportExtension.php index 4efcaace..38163886 100644 --- a/src/Wallabag/ImportBundle/DependencyInjection/WallabagImportExtension.php +++ b/src/Wallabag/ImportBundle/DependencyInjection/WallabagImportExtension.php @@ -13,6 +13,7 @@ class WallabagImportExtension extends Extension { $configuration = new Configuration(); $config = $this->processConfiguration($configuration, $configs); + $container->setParameter('wallabag_import.allow_mimetypes', $config['allow_mimetypes']); $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load('services.yml'); diff --git a/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php b/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php new file mode 100644 index 00000000..5d894318 --- /dev/null +++ b/src/Wallabag/ImportBundle/Form/Type/UploadImportType.php @@ -0,0 +1,29 @@ +add('file', 'file') + ->add('save', 'submit') + ; + } + + public function getDefaultOptions(array $options) + { + return array( + 'csrf_protection' => false, + ); + } + + public function getName() + { + return 'upload_import_file'; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index fda21f2d..ee759a52 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig @@ -13,4 +13,30 @@ + + +
+
+
+
+
+
+ {{ form_errors(form.import) }} +
+
+

{% trans %}Please select your wallabag export and click on the below button to upload and import it.{% endtrans %}

+ {{ form_errors(form.import.file) }} + {{ form_widget(form.import.file) }} +
+
+ + +
+
+
+
+
+
{% endblock %} -- cgit v1.2.3