From e2f3800ccb884682547769d9e4b5d6b7cafe4e07 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Fri, 24 Feb 2017 11:27:03 +0100 Subject: Add Clean Duplicates Command --- .../CoreBundle/Command/CleanDuplicatesCommand.php | 119 +++++++++++++++++++++ .../CoreBundle/Repository/EntryRepository.php | 13 +++ 2 files changed, 132 insertions(+) create mode 100644 src/Wallabag/CoreBundle/Command/CleanDuplicatesCommand.php diff --git a/src/Wallabag/CoreBundle/Command/CleanDuplicatesCommand.php b/src/Wallabag/CoreBundle/Command/CleanDuplicatesCommand.php new file mode 100644 index 00000000..65f35d8e --- /dev/null +++ b/src/Wallabag/CoreBundle/Command/CleanDuplicatesCommand.php @@ -0,0 +1,119 @@ +setName('wallabag:clean-duplicates') + ->setDescription('Cleans the database for duplicates') + ->setHelp('This command helps you to clean your articles list in case of duplicates') + ->addArgument( + 'username', + InputArgument::OPTIONAL, + 'User to clean' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $this->output = $output; + + $username = $input->getArgument('username'); + + if ($username) { + try { + $user = $this->getUser($username); + $this->cleanDuplicates($user); + } catch (NoResultException $e) { + $output->writeln(sprintf('User "%s" not found.', $username)); + + return 1; + } + } else { + $users = $this->getDoctrine()->getRepository('WallabagUserBundle:User')->findAll(); + + $output->writeln(sprintf('Cleaning through %d user accounts', count($users))); + + foreach ($users as $user) { + $output->writeln(sprintf('Processing user %s', $user->getUsername())); + $this->cleanDuplicates($user); + } + $output->writeln(sprintf('Finished cleaning. %d duplicates found in total', $this->duplicates)); + } + + return 0; + } + + /** + * @param User $user + */ + private function cleanDuplicates(User $user) + { + $em = $this->getContainer()->get('doctrine.orm.entity_manager'); + $repo = $this->getDoctrine()->getRepository('WallabagCoreBundle:Entry'); + + $entries = $repo->getAllEntriesIdAndUrl($user->getId()); + + $duplicatesCount = 0; + $urls = []; + foreach ($entries as $entry) { + $url = $this->similarUrl($entry['url']); + + /* @var $entry Entry */ + if (in_array($url, $urls)) { + ++$duplicatesCount; + + $em->remove($repo->find($entry['id'])); + $em->flush(); // Flushing at the end of the loop would require the instance not being online + } else { + $urls[] = $entry['url']; + } + } + + $this->duplicates += $duplicatesCount; + + $this->output->writeln(sprintf('Cleaned %d duplicates for user %s', $duplicatesCount, $user->getUserName())); + } + + private function similarUrl($url) + { + if (in_array(substr($url, -1), ['/', '#'])) { // get rid of "/" and "#" and the end of urls + return substr($url, 0, strlen($url)); + } + + return $url; + } + + /** + * Fetches a user from its username. + * + * @param string $username + * + * @return \Wallabag\UserBundle\Entity\User + */ + private function getUser($username) + { + return $this->getDoctrine()->getRepository('WallabagUserBundle:User')->findOneByUserName($username); + } + + private function getDoctrine() + { + return $this->getContainer()->get('doctrine'); + } +} diff --git a/src/Wallabag/CoreBundle/Repository/EntryRepository.php b/src/Wallabag/CoreBundle/Repository/EntryRepository.php index 1f22e901..5e7b0d3a 100644 --- a/src/Wallabag/CoreBundle/Repository/EntryRepository.php +++ b/src/Wallabag/CoreBundle/Repository/EntryRepository.php @@ -379,4 +379,17 @@ class EntryRepository extends EntityRepository ->setParameter('userId', $userId) ->execute(); } + + /** + * Get id and url from all entries + * Used for the clean-duplicates command. + */ + public function getAllEntriesIdAndUrl($userId) + { + $qb = $this->createQueryBuilder('e') + ->select('e.id, e.url') + ->where('e.user = :userid')->setParameter(':userid', $userId); + + return $qb->getQuery()->getArrayResult(); + } } -- cgit v1.2.3 From 3d57d625f88203ca526adf7729b93237ecd13242 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Thu, 30 Mar 2017 17:02:10 +0200 Subject: Add basic tests Signed-off-by: Thomas Citharel --- .../Command/CleanDuplicatesCommandTest.php | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php diff --git a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php new file mode 100644 index 00000000..9939d43c --- /dev/null +++ b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php @@ -0,0 +1,59 @@ +getClient()->getKernel()); + $application->add(new CleanDuplicatesCommand()); + + $command = $application->find('wallabag:clean-duplicates'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + ]); + + $this->assertContains('Cleaning through 3 user accounts', $tester->getDisplay()); + $this->assertContains('Finished cleaning. 0 duplicates found in total', $tester->getDisplay()); + } + + public function testRunTagAllCommandWithBadUsername() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new CleanDuplicatesCommand()); + + $command = $application->find('wallabag:clean-duplicates'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'username' => 'unknown', + ]); + + $this->assertContains('User "unknown" not found', $tester->getDisplay()); + } + + public function testRunTagAllCommandForUser() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new CleanDuplicatesCommand()); + + $command = $application->find('wallabag:clean-duplicates'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'username' => 'admin', + ]); + + $this->assertContains('Cleaned 0 duplicates for user admin', $tester->getDisplay()); + } +} -- cgit v1.2.3 From c613df0e25e9628a465758388c440da069623fd4 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 1 May 2017 13:30:00 +0200 Subject: Add docs for cli commands Signed-off-by: Thomas Citharel --- docs/en/developer/console_commands.rst | 30 ++++++++++++++++++++++++++++++ docs/fr/developer/console_commands.rst | 30 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 docs/en/developer/console_commands.rst create mode 100644 docs/fr/developer/console_commands.rst diff --git a/docs/en/developer/console_commands.rst b/docs/en/developer/console_commands.rst new file mode 100644 index 00000000..85a8a092 --- /dev/null +++ b/docs/en/developer/console_commands.rst @@ -0,0 +1,30 @@ +Console Commands +================ + +wallabag has a number of CLI commands to manage a number of tasks. You can list all the commands by executing `bin/console` in the wallabag folder. + +Each command has a help accessible through `bin/console help %command%`. + +.. note:: + + If you're in a production environment, remember to add `-e prod` to each command. + +Notable commands +---------------- + +* `assets:install`: May be helpful if assets are missing. +* `cache:clear`: should be run after each update (included in `make update`). +* `doctrine:migrations:status`: Output the status of your database migrations. +* `fos:user:activate`: Manually activate an user. +* `fos:user:change-password`: Change a password for an user. +* `fos:user:create`: Create an user. +* `fos:user:deactivate`: Deactivate an user (not deleted). +* `fos:user:demote`: Removes a role from an user, typically admin rights. +* `fos:user:promote`: Adds a role to an user, typically admin rights. +* `rabbitmq:*`: May be useful if you're using RabbitMQ. +* `wallabag:clean-duplicates`: Removes all entry duplicates for one user or all users +* `wallabag:export`: Exports all entries for an user. You can choose the output path of the file. +* `wallabag:import`: Import entries to different formats to an user account. +* `wallabag:import:redis-worker`: Useful if you use Redis. +* `wallabag:install`: (re)Install wallabag +* `wallabag:tag:all`: Tag all entries for an user using his/her tagging rules. diff --git a/docs/fr/developer/console_commands.rst b/docs/fr/developer/console_commands.rst new file mode 100644 index 00000000..1b222b32 --- /dev/null +++ b/docs/fr/developer/console_commands.rst @@ -0,0 +1,30 @@ +Actions en ligne de commande +============================ + +wallabag a un certain nombre de commandes CLI pour effectuer des tâches. Vous pouvez lister toutes les commandes en exécutant `bin/console` dans le dossier d'installation de wallabag. + +Chaque commande a une aide correspondante accessible via `bin/console help %command%`. + +.. note:: + + Si vous êtes dans un environnement de production, souvenez-vous d'ajouter `-e prod` à chaque commande. + +Commandes notables +------------------ + +* `assets:install`: Peut-être utile si les *assets* sont manquants. +* `cache:clear`: doit être exécuté après chaque mise à jour (appelé dans `make update`). +* `doctrine:migrations:status`: Montre le statut de vos migrations de vos bases de données. +* `fos:user:activate`: Activer manuellement un utilisateur. +* `fos:user:change-password`: Changer le mot de passe pour un utilisateur. +* `fos:user:create`: Créer un utilisateur. +* `fos:user:deactivate`: Désactiver un utilisateur (non supprimé). +* `fos:user:demote`: Supprimer un rôle d'un utilisateur, typiquement les droits d'administration. +* `fos:user:promote`: Ajoute un rôle à un utilisateur, typiquement les droits d'administration. +* `rabbitmq:*`: Peut-être utile si vous utilisez RabbitMQ. +* `wallabag:clean-duplicates`: Supprime tous les articles dupliqués pour un utilisateur ou bien tous. +* `wallabag:export`: Exporte tous les articles pour un utilisateur. Vous pouvez choisir le chemin du fichier exporté. +* `wallabag:import`: Importe les articles en différents formats dans un compte utilisateur. +* `wallabag:import:redis-worker`: Utile si vous utilisez Redis. +* `wallabag:install`: (ré)Installer wallabag +* `wallabag:tag:all`: Tagger tous les articles pour un utilisateur ou une utilisatrice en utilisant ses règles de tags automatiques. -- cgit v1.2.3 From d09fe4d233477d5cb9bfc613799b05a7ca14e270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 5 May 2017 14:33:36 +0200 Subject: Added test for deduplication --- .../CoreBundle/Repository/EntryRepository.php | 19 +++++++++ .../Command/CleanDuplicatesCommandTest.php | 47 +++++++++++++++++++++- .../CoreBundle/Command/ExportCommandTest.php | 2 +- .../CoreBundle/Helper/ContentProxyTest.php | 2 +- 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/Wallabag/CoreBundle/Repository/EntryRepository.php b/src/Wallabag/CoreBundle/Repository/EntryRepository.php index 5e7b0d3a..2e03fa19 100644 --- a/src/Wallabag/CoreBundle/Repository/EntryRepository.php +++ b/src/Wallabag/CoreBundle/Repository/EntryRepository.php @@ -392,4 +392,23 @@ class EntryRepository extends EntityRepository return $qb->getQuery()->getArrayResult(); } + + /** + * Find all entries by url and owner. + * + * @param $url + * @param $userId + * + * @return array + */ + public function findAllByUrlAndUserId($url, $userId) + { + $res = $this->createQueryBuilder('e') + ->where('e.url = :url')->setParameter('url', urldecode($url)) + ->andWhere('e.user = :user_id')->setParameter('user_id', $userId) + ->getQuery() + ->getResult(); + + return $res; + } } diff --git a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php index 9939d43c..1f5921d2 100644 --- a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php @@ -6,10 +6,11 @@ use Symfony\Bundle\FrameworkBundle\Console\Application; use Symfony\Component\Console\Tester\CommandTester; use Wallabag\CoreBundle\Command\CleanDuplicatesCommand; use Tests\Wallabag\CoreBundle\WallabagCoreTestCase; +use Wallabag\CoreBundle\Entity\Entry; class CleanDuplicatesCommandTest extends WallabagCoreTestCase { - public function testRunTagAllCommandForAll() + public function testTagAll() { $application = new Application($this->getClient()->getKernel()); $application->add(new CleanDuplicatesCommand()); @@ -56,4 +57,48 @@ class CleanDuplicatesCommandTest extends WallabagCoreTestCase $this->assertContains('Cleaned 0 duplicates for user admin', $tester->getDisplay()); } + + public function testDuplicate() + { + $url = 'http://www.lemonde.fr/sport/visuel/2017/05/05/rondelle-prison-blanchissage-comprendre-le-hockey-sur-glace_5122587_3242.html'; + $client = $this->getClient(); + $em = $client->getContainer()->get('doctrine.orm.entity_manager'); + + $this->logInAs('admin'); + + $nbEntries = $em->getRepository('WallabagCoreBundle:Entry')->findAllByUrlAndUserId($url, $this->getLoggedInUserId()); + $this->assertCount(0, $nbEntries); + + $user = $em->getRepository('WallabagUserBundle:User')->findOneById($this->getLoggedInUserId()); + + $entry1 = new Entry($user); + $entry1->setUrl($url); + + $entry2 = new Entry($user); + $entry2->setUrl($url); + + $em->persist($entry1); + $em->persist($entry2); + + $em->flush(); + + $nbEntries = $em->getRepository('WallabagCoreBundle:Entry')->findAllByUrlAndUserId($url, $this->getLoggedInUserId()); + $this->assertCount(2, $nbEntries); + + $application = new Application($this->getClient()->getKernel()); + $application->add(new CleanDuplicatesCommand()); + + $command = $application->find('wallabag:clean-duplicates'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'username' => 'admin', + ]); + + $this->assertContains('Cleaned 1 duplicates for user admin', $tester->getDisplay()); + + $nbEntries = $em->getRepository('WallabagCoreBundle:Entry')->findAllByUrlAndUserId($url, $this->getLoggedInUserId()); + $this->assertCount(1, $nbEntries); + } } diff --git a/tests/Wallabag/CoreBundle/Command/ExportCommandTest.php b/tests/Wallabag/CoreBundle/Command/ExportCommandTest.php index 6798c5d7..b21f3318 100644 --- a/tests/Wallabag/CoreBundle/Command/ExportCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/ExportCommandTest.php @@ -70,7 +70,7 @@ class ExportCommandTest extends WallabagCoreTestCase $tester->execute([ 'command' => $command->getName(), 'username' => 'admin', - 'filepath' => 'specialexport.json' + 'filepath' => 'specialexport.json', ]); $this->assertFileExists('specialexport.json'); diff --git a/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php b/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php index 5956b502..8abb1bbb 100644 --- a/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php +++ b/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php @@ -111,7 +111,7 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase $this->assertEquals('http://domain.io', $entry->getUrl()); $this->assertEquals('my title', $entry->getTitle()); - $this->assertEquals($this->fetchingErrorMessage . '

But we found a short description:

desc', $entry->getContent()); + $this->assertEquals($this->fetchingErrorMessage.'

But we found a short description:

desc', $entry->getContent()); $this->assertEmpty($entry->getPreviewPicture()); $this->assertEmpty($entry->getLanguage()); $this->assertEmpty($entry->getHttpStatus()); -- cgit v1.2.3 From 89f108b45ae94cd827595461b39f869111092579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 5 May 2017 14:54:03 +0200 Subject: Fixed @j0k3r review --- src/Wallabag/CoreBundle/Repository/EntryRepository.php | 4 +--- tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Wallabag/CoreBundle/Repository/EntryRepository.php b/src/Wallabag/CoreBundle/Repository/EntryRepository.php index 2e03fa19..6972e974 100644 --- a/src/Wallabag/CoreBundle/Repository/EntryRepository.php +++ b/src/Wallabag/CoreBundle/Repository/EntryRepository.php @@ -403,12 +403,10 @@ class EntryRepository extends EntityRepository */ public function findAllByUrlAndUserId($url, $userId) { - $res = $this->createQueryBuilder('e') + return $this->createQueryBuilder('e') ->where('e.url = :url')->setParameter('url', urldecode($url)) ->andWhere('e.user = :user_id')->setParameter('user_id', $userId) ->getQuery() ->getResult(); - - return $res; } } diff --git a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php index 1f5921d2..688cc388 100644 --- a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php @@ -10,7 +10,7 @@ use Wallabag\CoreBundle\Entity\Entry; class CleanDuplicatesCommandTest extends WallabagCoreTestCase { - public function testTagAll() + public function testRunCleanDuplicates() { $application = new Application($this->getClient()->getKernel()); $application->add(new CleanDuplicatesCommand()); @@ -26,7 +26,7 @@ class CleanDuplicatesCommandTest extends WallabagCoreTestCase $this->assertContains('Finished cleaning. 0 duplicates found in total', $tester->getDisplay()); } - public function testRunTagAllCommandWithBadUsername() + public function testRunCleanDuplicatesCommandWithBadUsername() { $application = new Application($this->getClient()->getKernel()); $application->add(new CleanDuplicatesCommand()); @@ -42,7 +42,7 @@ class CleanDuplicatesCommandTest extends WallabagCoreTestCase $this->assertContains('User "unknown" not found', $tester->getDisplay()); } - public function testRunTagAllCommandForUser() + public function testRunCleanDuplicatesCommandForUser() { $application = new Application($this->getClient()->getKernel()); $application->add(new CleanDuplicatesCommand()); -- cgit v1.2.3 From 4eeb29ff784934fa879dd87999e07c4c7626af8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 5 May 2017 15:20:58 +0200 Subject: Fixed test --- tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php index 688cc388..e6e57f30 100644 --- a/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/CleanDuplicatesCommandTest.php @@ -100,5 +100,9 @@ class CleanDuplicatesCommandTest extends WallabagCoreTestCase $nbEntries = $em->getRepository('WallabagCoreBundle:Entry')->findAllByUrlAndUserId($url, $this->getLoggedInUserId()); $this->assertCount(1, $nbEntries); + + $query = $em->createQuery('DELETE FROM Wallabag\CoreBundle\Entity\Entry e WHERE e.url = :url'); + $query->setParameter('url', $url); + $query->execute(); } } -- cgit v1.2.3