From 56ea1de99bff55ce4149ea8070eabf8183d6f4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 30 Sep 2015 11:43:25 +0200 Subject: first draft for json import, needed for wallabag v1 migration --- src/Wallabag/CoreBundle/Command/ImportCommand.php | 112 ++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 src/Wallabag/CoreBundle/Command/ImportCommand.php (limited to 'src/Wallabag/CoreBundle/Command') diff --git a/src/Wallabag/CoreBundle/Command/ImportCommand.php b/src/Wallabag/CoreBundle/Command/ImportCommand.php new file mode 100644 index 00000000..f27aeff7 --- /dev/null +++ b/src/Wallabag/CoreBundle/Command/ImportCommand.php @@ -0,0 +1,112 @@ +setName('import:json') + ->setDescription('Import entries from JSON file'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $now = new \DateTime(); + $output->writeln('Start : '.$now->format('d-m-Y G:i:s').' ---'); + + // Importing CSV on DB via Doctrine ORM + $this->import($input, $output); + + $now = new \DateTime(); + $output->writeln('End : '.$now->format('d-m-Y G:i:s').' ---'); + } + + protected function import(InputInterface $input, OutputInterface $output) + { + // Getting php array of data from CSV + $data = $this->get($input, $output); + + $em = $this->getContainer()->get('doctrine')->getManager(); + // Turning off doctrine default logs queries for saving memory + $em->getConnection()->getConfiguration()->setSQLLogger(null); + + // Define the size of record, the frequency for persisting the data and the current index of records + $size = count($data); + $batchSize = 20; + $i = 1; + + $progress = new ProgressBar($output, $size); + $progress->start(); + + $user = $em->getRepository('WallabagCoreBundle:User') + ->findOneById(1); + + foreach ($data as $object) { + $array = (array) $object; + $entry = $em->getRepository('WallabagCoreBundle:Entry') + ->findOneByUrl($array['url']); + + if (!is_object($entry)) { + $entry = new Entry($user); + $entry->setUrl($array['url']); + } + + $entry->setTitle($array['title']); + $entry->setArchived($array['is_read']); + $entry->setStarred($array['is_fav']); + $entry->setContent($array['content']); + $entry->setReadingTime(Utils::getReadingTime($array['content'])); + + $em->persist($entry); + + if (($i % $batchSize) === 0) { + $em->flush(); + $progress->advance($batchSize); + + $now = new \DateTime(); + $output->writeln(' of entries imported ... | '.$now->format('d-m-Y G:i:s')); + } + ++$i; + } + + $em->flush(); + $em->clear(); + $progress->finish(); + } + + protected function convert($filename) + { + if (!file_exists($filename) || !is_readable($filename)) { + return false; + } + + $header = null; + $data = array(); + + if (($handle = fopen($filename, 'r')) !== false) { + while (($row = fgets($handle)) !== false) { + $data = json_decode($row); + } + fclose($handle); + } + + return $data; + } + + protected function get(InputInterface $input, OutputInterface $output) + { + $fileName = 'web/uploads/import/import.json'; + $data = $this->convert($fileName); + + return $data; + } +} -- cgit v1.2.3 From a1bb1b3c2a0631ad41262ef92f6cce02c3d376bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 30 Sep 2015 14:22:38 +0200 Subject: userId is now set in parameters --- src/Wallabag/CoreBundle/Command/ImportCommand.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src/Wallabag/CoreBundle/Command') diff --git a/src/Wallabag/CoreBundle/Command/ImportCommand.php b/src/Wallabag/CoreBundle/Command/ImportCommand.php index f27aeff7..4a174202 100644 --- a/src/Wallabag/CoreBundle/Command/ImportCommand.php +++ b/src/Wallabag/CoreBundle/Command/ImportCommand.php @@ -3,6 +3,7 @@ namespace Wallabag\CoreBundle\Command; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; +use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Helper\ProgressBar; @@ -15,7 +16,12 @@ class ImportCommand extends ContainerAwareCommand { $this ->setName('import:json') - ->setDescription('Import entries from JSON file'); + ->setDescription('Import entries from JSON file') + ->addArgument( + 'userId', + InputArgument::REQUIRED, + 'user ID to populate' + ); } protected function execute(InputInterface $input, OutputInterface $output) @@ -32,6 +38,8 @@ class ImportCommand extends ContainerAwareCommand protected function import(InputInterface $input, OutputInterface $output) { + $userId = $input->getArgument('userId'); + // Getting php array of data from CSV $data = $this->get($input, $output); @@ -48,7 +56,7 @@ class ImportCommand extends ContainerAwareCommand $progress->start(); $user = $em->getRepository('WallabagCoreBundle:User') - ->findOneById(1); + ->findOneById($userId); foreach ($data as $object) { $array = (array) $object; -- cgit v1.2.3 From 8c3c77c1bd5c3763c127bfea52e908e77dc751b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 30 Sep 2015 15:10:46 +0200 Subject: create controller to launch import command --- src/Wallabag/CoreBundle/Command/ImportCommand.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src/Wallabag/CoreBundle/Command') diff --git a/src/Wallabag/CoreBundle/Command/ImportCommand.php b/src/Wallabag/CoreBundle/Command/ImportCommand.php index 4a174202..16c84229 100644 --- a/src/Wallabag/CoreBundle/Command/ImportCommand.php +++ b/src/Wallabag/CoreBundle/Command/ImportCommand.php @@ -3,6 +3,7 @@ namespace Wallabag\CoreBundle\Command; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; +use Symfony\Component\Config\Definition\Exception\Exception; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -15,7 +16,7 @@ class ImportCommand extends ContainerAwareCommand protected function configure() { $this - ->setName('import:json') + ->setName('wallabag:import') ->setDescription('Import entries from JSON file') ->addArgument( 'userId', @@ -38,8 +39,6 @@ class ImportCommand extends ContainerAwareCommand protected function import(InputInterface $input, OutputInterface $output) { - $userId = $input->getArgument('userId'); - // Getting php array of data from CSV $data = $this->get($input, $output); @@ -52,12 +51,16 @@ class ImportCommand extends ContainerAwareCommand $batchSize = 20; $i = 1; + $user = $em->getRepository('WallabagCoreBundle:User') + ->findOneById($input->getArgument('userId')); + + if (!is_object($user)) { + throw new Exception('User not found'); + } + $progress = new ProgressBar($output, $size); $progress->start(); - $user = $em->getRepository('WallabagCoreBundle:User') - ->findOneById($userId); - foreach ($data as $object) { $array = (array) $object; $entry = $em->getRepository('WallabagCoreBundle:Entry') @@ -112,8 +115,8 @@ class ImportCommand extends ContainerAwareCommand protected function get(InputInterface $input, OutputInterface $output) { - $fileName = 'web/uploads/import/import.json'; - $data = $this->convert($fileName); + $filename = __DIR__.'/../../../../web/uploads/import/'.$input->getArgument('userId').'.json'; + $data = $this->convert($filename); return $data; } -- cgit v1.2.3 From d275bdf4d36f90ff61f1e6a714c9ef64d210596f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Thu, 22 Oct 2015 16:57:56 +0200 Subject: form to upload file --- src/Wallabag/CoreBundle/Command/ImportCommand.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/Wallabag/CoreBundle/Command') diff --git a/src/Wallabag/CoreBundle/Command/ImportCommand.php b/src/Wallabag/CoreBundle/Command/ImportCommand.php index 16c84229..6be6f5e1 100644 --- a/src/Wallabag/CoreBundle/Command/ImportCommand.php +++ b/src/Wallabag/CoreBundle/Command/ImportCommand.php @@ -51,7 +51,7 @@ class ImportCommand extends ContainerAwareCommand $batchSize = 20; $i = 1; - $user = $em->getRepository('WallabagCoreBundle:User') + $user = $em->getRepository('WallabagUserBundle:User') ->findOneById($input->getArgument('userId')); if (!is_object($user)) { @@ -116,6 +116,7 @@ class ImportCommand extends ContainerAwareCommand protected function get(InputInterface $input, OutputInterface $output) { $filename = __DIR__.'/../../../../web/uploads/import/'.$input->getArgument('userId').'.json'; + $data = $this->convert($filename); return $data; -- cgit v1.2.3 From 77a7752a592af9ac821621a34d9955533baf40a0 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 30 Dec 2015 10:06:45 +0100 Subject: Update after previous merge PR #1443 was merged into this branch to handle all import type in the same place. --- src/Wallabag/CoreBundle/Command/ImportCommand.php | 124 ---------------------- 1 file changed, 124 deletions(-) delete mode 100644 src/Wallabag/CoreBundle/Command/ImportCommand.php (limited to 'src/Wallabag/CoreBundle/Command') diff --git a/src/Wallabag/CoreBundle/Command/ImportCommand.php b/src/Wallabag/CoreBundle/Command/ImportCommand.php deleted file mode 100644 index 6be6f5e1..00000000 --- a/src/Wallabag/CoreBundle/Command/ImportCommand.php +++ /dev/null @@ -1,124 +0,0 @@ -setName('wallabag:import') - ->setDescription('Import entries from JSON file') - ->addArgument( - 'userId', - InputArgument::REQUIRED, - 'user ID to populate' - ); - } - - protected function execute(InputInterface $input, OutputInterface $output) - { - $now = new \DateTime(); - $output->writeln('Start : '.$now->format('d-m-Y G:i:s').' ---'); - - // Importing CSV on DB via Doctrine ORM - $this->import($input, $output); - - $now = new \DateTime(); - $output->writeln('End : '.$now->format('d-m-Y G:i:s').' ---'); - } - - protected function import(InputInterface $input, OutputInterface $output) - { - // Getting php array of data from CSV - $data = $this->get($input, $output); - - $em = $this->getContainer()->get('doctrine')->getManager(); - // Turning off doctrine default logs queries for saving memory - $em->getConnection()->getConfiguration()->setSQLLogger(null); - - // Define the size of record, the frequency for persisting the data and the current index of records - $size = count($data); - $batchSize = 20; - $i = 1; - - $user = $em->getRepository('WallabagUserBundle:User') - ->findOneById($input->getArgument('userId')); - - if (!is_object($user)) { - throw new Exception('User not found'); - } - - $progress = new ProgressBar($output, $size); - $progress->start(); - - foreach ($data as $object) { - $array = (array) $object; - $entry = $em->getRepository('WallabagCoreBundle:Entry') - ->findOneByUrl($array['url']); - - if (!is_object($entry)) { - $entry = new Entry($user); - $entry->setUrl($array['url']); - } - - $entry->setTitle($array['title']); - $entry->setArchived($array['is_read']); - $entry->setStarred($array['is_fav']); - $entry->setContent($array['content']); - $entry->setReadingTime(Utils::getReadingTime($array['content'])); - - $em->persist($entry); - - if (($i % $batchSize) === 0) { - $em->flush(); - $progress->advance($batchSize); - - $now = new \DateTime(); - $output->writeln(' of entries imported ... | '.$now->format('d-m-Y G:i:s')); - } - ++$i; - } - - $em->flush(); - $em->clear(); - $progress->finish(); - } - - protected function convert($filename) - { - if (!file_exists($filename) || !is_readable($filename)) { - return false; - } - - $header = null; - $data = array(); - - if (($handle = fopen($filename, 'r')) !== false) { - while (($row = fgets($handle)) !== false) { - $data = json_decode($row); - } - fclose($handle); - } - - return $data; - } - - protected function get(InputInterface $input, OutputInterface $output) - { - $filename = __DIR__.'/../../../../web/uploads/import/'.$input->getArgument('userId').'.json'; - - $data = $this->convert($filename); - - return $data; - } -} -- cgit v1.2.3 From 8eedc8cfacc07e998f6f0bcdfe5b76496a215ea2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 3 Jan 2016 10:59:55 +0100 Subject: Few phpDoc fix And some little mistakes --- src/Wallabag/CoreBundle/Command/InstallCommand.php | 1 - 1 file changed, 1 deletion(-) (limited to 'src/Wallabag/CoreBundle/Command') diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 85c4ee90..e791d4dd 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -11,7 +11,6 @@ use Symfony\Component\Console\Output\NullOutput; use Symfony\Component\Console\Question\Question; use Symfony\Component\Console\Question\ConfirmationQuestion; use Symfony\Component\Console\Helper\Table; -use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Entity\Config; class InstallCommand extends ContainerAwareCommand -- cgit v1.2.3