From 9c2b2aae70b06411336e6eb6ac43b3ebd30dc38c Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 1 Apr 2019 11:50:33 +0200 Subject: [PATCH] Keep url in exists endpoint - Add migration - Use md5 instead of sha512 (we don't need security here, just a hash) - Update tests --- .../Version20190401105353.php | 44 +++++++++++ .../Controller/EntryRestController.php | 55 +++++++------- .../Command/GenerateUrlHashesCommand.php | 19 +++-- .../CoreBundle/DataFixtures/EntryFixtures.php | 2 +- src/Wallabag/CoreBundle/Entity/Entry.php | 4 +- .../CoreBundle/Repository/EntryRepository.php | 24 ++++++ .../Controller/EntryRestControllerTest.php | 75 +++++++++---------- .../Command/GenerateUrlHashesCommandTest.php | 8 +- 8 files changed, 154 insertions(+), 77 deletions(-) create mode 100644 app/DoctrineMigrations/Version20190401105353.php diff --git a/app/DoctrineMigrations/Version20190401105353.php b/app/DoctrineMigrations/Version20190401105353.php new file mode 100644 index 00000000..4afc8b15 --- /dev/null +++ b/app/DoctrineMigrations/Version20190401105353.php @@ -0,0 +1,44 @@ +getTable($this->getTable('entry')); + + $this->skipIf($entryTable->hasColumn('hashed_url'), 'It seems that you already played this migration.'); + + $entryTable->addColumn('hashed_url', 'text', [ + 'length' => 32, + 'notnull' => false, + ]); + + // sqlite doesn't have the MD5 function by default + if ('sqlite' !== $this->connection->getDatabasePlatform()->getName()) { + $this->addSql('UPDATE ' . $this->getTable('entry') . ' SET hashed_url = MD5(url)'); + } + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + $entryTable = $schema->getTable($this->getTable('entry')); + + $this->skipIf(!$entryTable->hasColumn('hashed_url'), 'It seems that you already played this migration.'); + + $entryTable->dropColumn('hashed_url'); + } +} diff --git a/src/Wallabag/ApiBundle/Controller/EntryRestController.php b/src/Wallabag/ApiBundle/Controller/EntryRestController.php index 26746f7d..0ecf1a0e 100644 --- a/src/Wallabag/ApiBundle/Controller/EntryRestController.php +++ b/src/Wallabag/ApiBundle/Controller/EntryRestController.php @@ -27,10 +27,10 @@ class EntryRestController extends WallabagRestController * @ApiDoc( * parameters={ * {"name"="return_id", "dataType"="string", "required"=false, "format"="1 or 0", "description"="Set 1 if you want to retrieve ID in case entry(ies) exists, 0 by default"}, - * {"name"="url", "dataType"="string", "required"=true, "format"="An url", "description"="Url to check if it exists"}, - * {"name"="urls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="Urls (as an array) to check if it exists"} - * {"name"="hashedurl", "dataType"="string", "required"=true, "format"="An url", "description"="Md5 url to check if it exists"}, - * {"name"="hashedurls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="Md5 urls (as an array) to check if it exists"} + * {"name"="url", "dataType"="string", "required"=true, "format"="An url", "description"="DEPRECATED, use hashed_url instead"}, + * {"name"="urls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="DEPRECATED, use hashed_urls instead"}, + * {"name"="hashed_url", "dataType"="string", "required"=true, "format"="An url", "description"="Md5 url to check if it exists"}, + * {"name"="hashed_urls", "dataType"="string", "required"=false, "format"="An array of urls (?urls[]=http...&urls[]=http...)", "description"="Md5 urls (as an array) to check if it exists"} * } * ) * @@ -39,22 +39,18 @@ class EntryRestController extends WallabagRestController public function getEntriesExistsAction(Request $request) { $this->validateAuthentication(); + $repo = $this->getDoctrine()->getRepository('WallabagCoreBundle:Entry'); $returnId = (null === $request->query->get('return_id')) ? false : (bool) $request->query->get('return_id'); - $urls = $request->query->get('urls', []); - $hashedUrls = $request->query->get('hashedurls', []); + $urls = $request->query->get('urls', []); + $hashedUrls = $request->query->get('hashed_urls', []); // handle multiple urls first if (!empty($hashedUrls)) { $results = []; foreach ($hashedUrls as $hashedUrl) { - $res = $this->getDoctrine() - ->getRepository('WallabagCoreBundle:Entry') - ->findOneBy([ - 'hashedUrl' => $hashedUrl, - 'user' => $this->getUser()->getId(), - ]); + $res = $repo->findByHashedUrlAndUserId($hashedUrl, $this->getUser()->getId()); // $results[$url] = $this->returnExistInformation($res, $returnId); $results[$hashedUrl] = $this->returnExistInformation($res, $returnId); @@ -63,24 +59,33 @@ class EntryRestController extends WallabagRestController return $this->sendResponse($results); } + // @deprecated, to be remove in 3.0 + if (!empty($urls)) { + $results = []; + foreach ($urls as $url) { + $res = $repo->findByUrlAndUserId($url, $this->getUser()->getId()); + + $results[$url] = $this->returnExistInformation($res, $returnId); + } + + return $this->sendResponse($results); + } + // let's see if it is a simple url? - $hashedUrl = $request->query->get('hashedurl', ''); + $url = $request->query->get('url', ''); + $hashedUrl = $request->query->get('hashed_url', ''); - // if (empty($url)) { - // throw $this->createAccessDeniedException('URL is empty?, logged user id: ' . $this->getUser()->getId()); - // } + if (empty($url) && empty($hashedUrl)) { + throw $this->createAccessDeniedException('URL is empty?, logged user id: ' . $this->getUser()->getId()); + } - if (empty($hashedUrl)) { - throw $this->createAccessDeniedException('URL is empty?, logged user id: '.$this->getUser()->getId()); + $method = 'findByUrlAndUserId'; + if (!empty($hashedUrl)) { + $method = 'findByHashedUrlAndUserId'; + $url = $hashedUrl; } - $res = $this->getDoctrine() - ->getRepository('WallabagCoreBundle:Entry') - // ->findByUrlAndUserId($url, $this->getUser()->getId()); - ->findOneBy([ - 'hashedUrl' => $hashedUrl, - 'user' => $this->getUser()->getId(), - ]); + $res = $repo->$method($url, $this->getUser()->getId()); return $this->sendResponse(['exists' => $this->returnExistInformation($res, $returnId)]); } diff --git a/src/Wallabag/CoreBundle/Command/GenerateUrlHashesCommand.php b/src/Wallabag/CoreBundle/Command/GenerateUrlHashesCommand.php index fe2644f2..fb598390 100644 --- a/src/Wallabag/CoreBundle/Command/GenerateUrlHashesCommand.php +++ b/src/Wallabag/CoreBundle/Command/GenerateUrlHashesCommand.php @@ -45,13 +45,13 @@ class GenerateUrlHashesCommand extends ContainerAwareCommand } else { $users = $this->getDoctrine()->getRepository('WallabagUserBundle:User')->findAll(); - $output->writeln(sprintf('Generating hashed urls for the %d user account entries', count($users))); + $output->writeln(sprintf('Generating hashed urls for "%d" users', \count($users))); foreach ($users as $user) { - $output->writeln(sprintf('Processing user %s', $user->getUsername())); + $output->writeln(sprintf('Processing user: %s', $user->getUsername())); $this->generateHashedUrls($user); } - $output->writeln(sprintf('Finished generated hashed urls')); + $output->writeln('Finished generated hashed urls'); } return 0; @@ -67,13 +67,20 @@ class GenerateUrlHashesCommand extends ContainerAwareCommand $entries = $repo->findByUser($user->getId()); + $i = 1; foreach ($entries as $entry) { - $entry->setHashedUrl(hash('sha512', $entry->getUrl())); + $entry->setHashedUrl(hash('md5', $entry->getUrl())); $em->persist($entry); - $em->flush(); + + if (0 === ($i % 20)) { + $em->flush(); + } + ++$i; } - $this->output->writeln(sprintf('Generated hashed urls for user %s', $user->getUserName())); + $em->flush(); + + $this->output->writeln(sprintf('Generated hashed urls for user: %s', $user->getUserName())); } /** diff --git a/src/Wallabag/CoreBundle/DataFixtures/EntryFixtures.php b/src/Wallabag/CoreBundle/DataFixtures/EntryFixtures.php index 9c10500d..1b18cad6 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/EntryFixtures.php +++ b/src/Wallabag/CoreBundle/DataFixtures/EntryFixtures.php @@ -30,7 +30,6 @@ class EntryFixtures extends Fixture implements DependentFixtureInterface 'entry2' => [ 'user' => 'admin-user', 'url' => 'http://0.0.0.0/entry2', - 'hashed_url' => hash('md5', 'http://0.0.0.0/entry2'), 'reading_time' => 1, 'domain' => 'domain.io', 'mime' => 'text/html', @@ -90,6 +89,7 @@ class EntryFixtures extends Fixture implements DependentFixtureInterface foreach ($entries as $reference => $item) { $entry = new Entry($this->getReference($item['user'])); $entry->setUrl($item['url']); + $entry->setHashedUrl(hash('md5', $item['url'])); $entry->setReadingTime($item['reading_time']); $entry->setDomainName($item['domain']); $entry->setMimetype($item['mime']); diff --git a/src/Wallabag/CoreBundle/Entity/Entry.php b/src/Wallabag/CoreBundle/Entity/Entry.php index 17a1ed58..a04f101f 100644 --- a/src/Wallabag/CoreBundle/Entity/Entry.php +++ b/src/Wallabag/CoreBundle/Entity/Entry.php @@ -26,7 +26,7 @@ use Wallabag\UserBundle\Entity\User; * indexes={ * @ORM\Index(name="created_at", columns={"created_at"}), * @ORM\Index(name="uid", columns={"uid"}), - * @ORM\Index(name="hashedurl", columns={"hashedurl"}) + * @ORM\Index(name="hashed_url", columns={"hashed_url"}) * } * ) * @ORM\HasLifecycleCallbacks() @@ -79,7 +79,7 @@ class Entry /** * @var string * - * @ORM\Column(name="hashedurl", type="text", nullable=true) + * @ORM\Column(name="hashed_url", type="string", length=32, nullable=true) */ private $hashedUrl; diff --git a/src/Wallabag/CoreBundle/Repository/EntryRepository.php b/src/Wallabag/CoreBundle/Repository/EntryRepository.php index 45366623..0c175abb 100644 --- a/src/Wallabag/CoreBundle/Repository/EntryRepository.php +++ b/src/Wallabag/CoreBundle/Repository/EntryRepository.php @@ -346,6 +346,30 @@ class EntryRepository extends EntityRepository return false; } + /** + * Find an entry by its hashed url and its owner. + * If it exists, return the entry otherwise return false. + * + * @param $hashedUrl + * @param $userId + * + * @return Entry|bool + */ + public function findByHashedUrlAndUserId($hashedUrl, $userId) + { + $res = $this->createQueryBuilder('e') + ->where('e.hashedUrl = :hashed_url')->setParameter('hashed_url', urldecode($hashedUrl)) + ->andWhere('e.user = :user_id')->setParameter('user_id', $userId) + ->getQuery() + ->getResult(); + + if (\count($res)) { + return current($res); + } + + return false; + } + /** * Count all entries for a user. * diff --git a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php index 8d96d7b8..fc4dc9d9 100644 --- a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php +++ b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php @@ -971,40 +971,42 @@ class EntryRestControllerTest extends WallabagApiTestCase $this->assertGreaterThanOrEqual($now->getTimestamp(), (new \DateTime($content['starred_at']))->getTimestamp()); } - public function testGetEntriesExistsWithReturnId() + public function dataForEntriesExistWithUrl() { - $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2&return_id=1'); + $url = hash('md5', 'http://0.0.0.0/entry2'); - $this->assertSame(200, $this->client->getResponse()->getStatusCode()); - - $content = json_decode($this->client->getResponse()->getContent(), true); - - // it returns a database id, we don't know it, so we only check it's greater than the lowest possible value - $this->assertGreaterThan(1, $content['exists']); - } - - public function testGetEntriesExistsWithoutReturnId() - { - $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2'); - - $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2')); - - $this->assertSame(200, $this->client->getResponse()->getStatusCode()); - - $content = json_decode($this->client->getResponse()->getContent(), true); - - $this->assertTrue($content['exists']); + return [ + 'with_id' => [ + 'url' => '/api/entries/exists?url=http://0.0.0.0/entry2&return_id=1', + 'expectedValue' => 2, + ], + 'without_id' => [ + 'url' => '/api/entries/exists?url=http://0.0.0.0/entry2', + 'expectedValue' => true, + ], + 'hashed_url_with_id' => [ + 'url' => '/api/entries/exists?hashed_url=' . $url . '&return_id=1', + 'expectedValue' => 2, + ], + 'hashed_url_without_id' => [ + 'url' => '/api/entries/exists?hashed_url=' . $url . '', + 'expectedValue' => true, + ], + ]; } - public function testGetEntriesExistsWithHash() + /** + * @dataProvider dataForEntriesExistWithUrl + */ + public function testGetEntriesExists($url, $expectedValue) { - $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2')); + $this->client->request('GET', $url); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertSame(2, $content['exists']); + $this->assertSame($expectedValue, $content['exists']); } public function testGetEntriesExistsWithManyUrls() @@ -1045,42 +1047,37 @@ class EntryRestControllerTest extends WallabagApiTestCase { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; - $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2) . '&return_id=1'); + $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2) . '&return_id=1'); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertArrayHasKey($url1, $content); - $this->assertArrayHasKey($url2, $content); - $this->assertSame(2, $content[$url1]); - $this->assertNull($content[$url2]); - $this->assertArrayHasKey(hash('md5', $url1), $content); $this->assertArrayHasKey(hash('md5', $url2), $content); - $this->assertEquals(2, $content[hash('md5', $url1)]); - $this->assertEquals(false, $content[hash('md5', $url2)]); + $this->assertSame(2, $content[hash('md5', $url1)]); + $this->assertNull($content[hash('md5', $url2)]); } public function testGetEntriesExistsWithManyUrlsHashedReturnBool() { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; - $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2)); + $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2)); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertArrayHasKey($url1, $content); - $this->assertArrayHasKey($url2, $content); - $this->assertTrue($content[$url1]); - $this->assertFalse($content[$url2]); + $this->assertArrayHasKey(hash('md5', $url1), $content); + $this->assertArrayHasKey(hash('md5', $url2), $content); + $this->assertTrue($content[hash('md5', $url1)]); + $this->assertFalse($content[hash('md5', $url2)]); } public function testGetEntriesExistsWhichDoesNotExists() { - $this->client->request('GET', '/api/entries/exists?hashedurl='.hash('md5','http://google.com/entry2')); + $this->client->request('GET', '/api/entries/exists?hashed_url=' . hash('md5', 'http://google.com/entry2')); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); @@ -1091,7 +1088,7 @@ class EntryRestControllerTest extends WallabagApiTestCase public function testGetEntriesExistsWithNoUrl() { - $this->client->request('GET', '/api/entries/exists?hashedurl='); + $this->client->request('GET', '/api/entries/exists?hashed_url='); $this->assertSame(403, $this->client->getResponse()->getStatusCode()); } diff --git a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php index 8ca772cb..cc1e3fbc 100644 --- a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php @@ -22,7 +22,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase 'command' => $command->getName(), ]); - $this->assertContains('Generating hashed urls for the 3 user account entries', $tester->getDisplay()); + $this->assertContains('Generating hashed urls for "3" users', $tester->getDisplay()); $this->assertContains('Finished generated hashed urls', $tester->getDisplay()); } @@ -55,7 +55,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase 'username' => 'admin', ]); - $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay()); + $this->assertContains('Generated hashed urls for user: admin', $tester->getDisplay()); } public function testGenerateUrls() @@ -88,11 +88,11 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase 'username' => 'admin', ]); - $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay()); + $this->assertContains('Generated hashed urls for user: admin', $tester->getDisplay()); $entry = $em->getRepository('WallabagCoreBundle:Entry')->findOneByUrl($url); - $this->assertEquals($entry->getHashedUrl(), hash('sha512', $url)); + $this->assertSame($entry->getHashedUrl(), hash('md5', $url)); $query = $em->createQuery('DELETE FROM Wallabag\CoreBundle\Entity\Entry e WHERE e.url = :url'); $query->setParameter('url', $url); -- 2.41.0