From bfe02a0b481055bb4e799200c8daa9a0ad987c71 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Sun, 28 May 2017 14:53:04 +0200 Subject: Hash the urls to check if they exist Signed-off-by: Thomas Citharel --- .../Controller/EntryRestControllerTest.php | 55 ++++++++++- .../Command/GenerateUrlHashesCommandTest.php | 101 +++++++++++++++++++++ 2 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php (limited to 'tests') diff --git a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php index 2151f587..8d96d7b8 100644 --- a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php +++ b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php @@ -987,6 +987,8 @@ class EntryRestControllerTest extends WallabagApiTestCase { $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2'); + $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2')); + $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); @@ -994,10 +996,22 @@ class EntryRestControllerTest extends WallabagApiTestCase $this->assertTrue($content['exists']); } + public function testGetEntriesExistsWithHash() + { + $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2')); + + $this->assertSame(200, $this->client->getResponse()->getStatusCode()); + + $content = json_decode($this->client->getResponse()->getContent(), true); + + $this->assertSame(2, $content['exists']); + } + public function testGetEntriesExistsWithManyUrls() { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; + $this->client->request('GET', '/api/entries/exists?urls[]=' . $url1 . '&urls[]=' . $url2 . '&return_id=1'); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); @@ -1027,9 +1041,46 @@ class EntryRestControllerTest extends WallabagApiTestCase $this->assertFalse($content[$url2]); } + public function testGetEntriesExistsWithManyUrlsHashed() + { + $url1 = 'http://0.0.0.0/entry2'; + $url2 = 'http://0.0.0.0/entry10'; + $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2) . '&return_id=1'); + + $this->assertSame(200, $this->client->getResponse()->getStatusCode()); + + $content = json_decode($this->client->getResponse()->getContent(), true); + + $this->assertArrayHasKey($url1, $content); + $this->assertArrayHasKey($url2, $content); + $this->assertSame(2, $content[$url1]); + $this->assertNull($content[$url2]); + + $this->assertArrayHasKey(hash('md5', $url1), $content); + $this->assertArrayHasKey(hash('md5', $url2), $content); + $this->assertEquals(2, $content[hash('md5', $url1)]); + $this->assertEquals(false, $content[hash('md5', $url2)]); + } + + public function testGetEntriesExistsWithManyUrlsHashedReturnBool() + { + $url1 = 'http://0.0.0.0/entry2'; + $url2 = 'http://0.0.0.0/entry10'; + $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2)); + + $this->assertSame(200, $this->client->getResponse()->getStatusCode()); + + $content = json_decode($this->client->getResponse()->getContent(), true); + + $this->assertArrayHasKey($url1, $content); + $this->assertArrayHasKey($url2, $content); + $this->assertTrue($content[$url1]); + $this->assertFalse($content[$url2]); + } + public function testGetEntriesExistsWhichDoesNotExists() { - $this->client->request('GET', '/api/entries/exists?url=http://google.com/entry2'); + $this->client->request('GET', '/api/entries/exists?hashedurl='.hash('md5','http://google.com/entry2')); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); @@ -1040,7 +1091,7 @@ class EntryRestControllerTest extends WallabagApiTestCase public function testGetEntriesExistsWithNoUrl() { - $this->client->request('GET', '/api/entries/exists?url='); + $this->client->request('GET', '/api/entries/exists?hashedurl='); $this->assertSame(403, $this->client->getResponse()->getStatusCode()); } diff --git a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php new file mode 100644 index 00000000..8ca772cb --- /dev/null +++ b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php @@ -0,0 +1,101 @@ +getClient()->getKernel()); + $application->add(new GenerateUrlHashesCommand()); + + $command = $application->find('wallabag:generate-hashed-urls'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + ]); + + $this->assertContains('Generating hashed urls for the 3 user account entries', $tester->getDisplay()); + $this->assertContains('Finished generated hashed urls', $tester->getDisplay()); + } + + public function testRunGenerateUrlHashesCommandWithBadUsername() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new GenerateUrlHashesCommand()); + + $command = $application->find('wallabag:generate-hashed-urls'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'username' => 'unknown', + ]); + + $this->assertContains('User "unknown" not found', $tester->getDisplay()); + } + + public function testRunGenerateUrlHashesCommandForUser() + { + $application = new Application($this->getClient()->getKernel()); + $application->add(new GenerateUrlHashesCommand()); + + $command = $application->find('wallabag:generate-hashed-urls'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'username' => 'admin', + ]); + + $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay()); + } + + public function testGenerateUrls() + { + $url = 'http://www.lemonde.fr/sport/visuel/2017/05/05/rondelle-prison-blanchissage-comprendre-le-hockey-sur-glace_5122587_3242.html'; + $client = $this->getClient(); + $em = $client->getContainer()->get('doctrine.orm.entity_manager'); + + $this->logInAs('admin'); + + $user = $em->getRepository('WallabagUserBundle:User')->findOneById($this->getLoggedInUserId()); + + $entry1 = new Entry($user); + $entry1->setUrl($url); + + $em->persist($entry1); + + $em->flush(); + + $this->assertNull($entry1->getHashedUrl()); + + $application = new Application($this->getClient()->getKernel()); + $application->add(new GenerateUrlHashesCommand()); + + $command = $application->find('wallabag:generate-hashed-urls'); + + $tester = new CommandTester($command); + $tester->execute([ + 'command' => $command->getName(), + 'username' => 'admin', + ]); + + $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay()); + + $entry = $em->getRepository('WallabagCoreBundle:Entry')->findOneByUrl($url); + + $this->assertEquals($entry->getHashedUrl(), hash('sha512', $url)); + + $query = $em->createQuery('DELETE FROM Wallabag\CoreBundle\Entity\Entry e WHERE e.url = :url'); + $query->setParameter('url', $url); + $query->execute(); + } +} -- cgit v1.2.3 From 9c2b2aae70b06411336e6eb6ac43b3ebd30dc38c Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 1 Apr 2019 11:50:33 +0200 Subject: Keep url in exists endpoint - Add migration - Use md5 instead of sha512 (we don't need security here, just a hash) - Update tests --- .../Controller/EntryRestControllerTest.php | 75 +++++++++++----------- .../Command/GenerateUrlHashesCommandTest.php | 8 +-- 2 files changed, 40 insertions(+), 43 deletions(-) (limited to 'tests') diff --git a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php index 8d96d7b8..fc4dc9d9 100644 --- a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php +++ b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php @@ -971,40 +971,42 @@ class EntryRestControllerTest extends WallabagApiTestCase $this->assertGreaterThanOrEqual($now->getTimestamp(), (new \DateTime($content['starred_at']))->getTimestamp()); } - public function testGetEntriesExistsWithReturnId() + public function dataForEntriesExistWithUrl() { - $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2&return_id=1'); + $url = hash('md5', 'http://0.0.0.0/entry2'); - $this->assertSame(200, $this->client->getResponse()->getStatusCode()); - - $content = json_decode($this->client->getResponse()->getContent(), true); - - // it returns a database id, we don't know it, so we only check it's greater than the lowest possible value - $this->assertGreaterThan(1, $content['exists']); - } - - public function testGetEntriesExistsWithoutReturnId() - { - $this->client->request('GET', '/api/entries/exists?url=http://0.0.0.0/entry2'); - - $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2')); - - $this->assertSame(200, $this->client->getResponse()->getStatusCode()); - - $content = json_decode($this->client->getResponse()->getContent(), true); - - $this->assertTrue($content['exists']); + return [ + 'with_id' => [ + 'url' => '/api/entries/exists?url=http://0.0.0.0/entry2&return_id=1', + 'expectedValue' => 2, + ], + 'without_id' => [ + 'url' => '/api/entries/exists?url=http://0.0.0.0/entry2', + 'expectedValue' => true, + ], + 'hashed_url_with_id' => [ + 'url' => '/api/entries/exists?hashed_url=' . $url . '&return_id=1', + 'expectedValue' => 2, + ], + 'hashed_url_without_id' => [ + 'url' => '/api/entries/exists?hashed_url=' . $url . '', + 'expectedValue' => true, + ], + ]; } - public function testGetEntriesExistsWithHash() + /** + * @dataProvider dataForEntriesExistWithUrl + */ + public function testGetEntriesExists($url, $expectedValue) { - $this->client->request('GET', '/api/entries/exists?hashedurl=' . hash('md5', 'http://0.0.0.0/entry2')); + $this->client->request('GET', $url); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertSame(2, $content['exists']); + $this->assertSame($expectedValue, $content['exists']); } public function testGetEntriesExistsWithManyUrls() @@ -1045,42 +1047,37 @@ class EntryRestControllerTest extends WallabagApiTestCase { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; - $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2) . '&return_id=1'); + $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2) . '&return_id=1'); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertArrayHasKey($url1, $content); - $this->assertArrayHasKey($url2, $content); - $this->assertSame(2, $content[$url1]); - $this->assertNull($content[$url2]); - $this->assertArrayHasKey(hash('md5', $url1), $content); $this->assertArrayHasKey(hash('md5', $url2), $content); - $this->assertEquals(2, $content[hash('md5', $url1)]); - $this->assertEquals(false, $content[hash('md5', $url2)]); + $this->assertSame(2, $content[hash('md5', $url1)]); + $this->assertNull($content[hash('md5', $url2)]); } public function testGetEntriesExistsWithManyUrlsHashedReturnBool() { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; - $this->client->request('GET', '/api/entries/exists?hashedurls[]='.hash('md5',$url1).'&hashedurls[]='.hash('md5',$url2)); + $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2)); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertArrayHasKey($url1, $content); - $this->assertArrayHasKey($url2, $content); - $this->assertTrue($content[$url1]); - $this->assertFalse($content[$url2]); + $this->assertArrayHasKey(hash('md5', $url1), $content); + $this->assertArrayHasKey(hash('md5', $url2), $content); + $this->assertTrue($content[hash('md5', $url1)]); + $this->assertFalse($content[hash('md5', $url2)]); } public function testGetEntriesExistsWhichDoesNotExists() { - $this->client->request('GET', '/api/entries/exists?hashedurl='.hash('md5','http://google.com/entry2')); + $this->client->request('GET', '/api/entries/exists?hashed_url=' . hash('md5', 'http://google.com/entry2')); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); @@ -1091,7 +1088,7 @@ class EntryRestControllerTest extends WallabagApiTestCase public function testGetEntriesExistsWithNoUrl() { - $this->client->request('GET', '/api/entries/exists?hashedurl='); + $this->client->request('GET', '/api/entries/exists?hashed_url='); $this->assertSame(403, $this->client->getResponse()->getStatusCode()); } diff --git a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php index 8ca772cb..cc1e3fbc 100644 --- a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php @@ -22,7 +22,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase 'command' => $command->getName(), ]); - $this->assertContains('Generating hashed urls for the 3 user account entries', $tester->getDisplay()); + $this->assertContains('Generating hashed urls for "3" users', $tester->getDisplay()); $this->assertContains('Finished generated hashed urls', $tester->getDisplay()); } @@ -55,7 +55,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase 'username' => 'admin', ]); - $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay()); + $this->assertContains('Generated hashed urls for user: admin', $tester->getDisplay()); } public function testGenerateUrls() @@ -88,11 +88,11 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase 'username' => 'admin', ]); - $this->assertContains('Generated hashed urls for user admin', $tester->getDisplay()); + $this->assertContains('Generated hashed urls for user: admin', $tester->getDisplay()); $entry = $em->getRepository('WallabagCoreBundle:Entry')->findOneByUrl($url); - $this->assertEquals($entry->getHashedUrl(), hash('sha512', $url)); + $this->assertSame($entry->getHashedUrl(), hash('md5', $url)); $query = $em->createQuery('DELETE FROM Wallabag\CoreBundle\Entity\Entry e WHERE e.url = :url'); $query->setParameter('url', $url); -- cgit v1.2.3 From 8a6456629814039cfc623cdb279bcba06dacff50 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 1 Apr 2019 13:51:57 +0200 Subject: Use a better index for hashed_url It'll most often be used in addition to the `user_id`. Also, automatically generate the hash when saving the url. Switch from `md5` to `sha1`. --- .../Controller/EntryRestControllerTest.php | 24 +++++++++++----------- .../Command/GenerateUrlHashesCommandTest.php | 5 +---- 2 files changed, 13 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php index fc4dc9d9..34de8ec8 100644 --- a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php +++ b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php @@ -973,7 +973,7 @@ class EntryRestControllerTest extends WallabagApiTestCase public function dataForEntriesExistWithUrl() { - $url = hash('md5', 'http://0.0.0.0/entry2'); + $url = hash('sha1', 'http://0.0.0.0/entry2'); return [ 'with_id' => [ @@ -1047,37 +1047,37 @@ class EntryRestControllerTest extends WallabagApiTestCase { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; - $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2) . '&return_id=1'); + $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('sha1', $url1) . '&hashed_urls[]=' . hash('sha1', $url2) . '&return_id=1'); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertArrayHasKey(hash('md5', $url1), $content); - $this->assertArrayHasKey(hash('md5', $url2), $content); - $this->assertSame(2, $content[hash('md5', $url1)]); - $this->assertNull($content[hash('md5', $url2)]); + $this->assertArrayHasKey(hash('sha1', $url1), $content); + $this->assertArrayHasKey(hash('sha1', $url2), $content); + $this->assertSame(2, $content[hash('sha1', $url1)]); + $this->assertNull($content[hash('sha1', $url2)]); } public function testGetEntriesExistsWithManyUrlsHashedReturnBool() { $url1 = 'http://0.0.0.0/entry2'; $url2 = 'http://0.0.0.0/entry10'; - $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('md5', $url1) . '&hashed_urls[]=' . hash('md5', $url2)); + $this->client->request('GET', '/api/entries/exists?hashed_urls[]=' . hash('sha1', $url1) . '&hashed_urls[]=' . hash('sha1', $url2)); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); $content = json_decode($this->client->getResponse()->getContent(), true); - $this->assertArrayHasKey(hash('md5', $url1), $content); - $this->assertArrayHasKey(hash('md5', $url2), $content); - $this->assertTrue($content[hash('md5', $url1)]); - $this->assertFalse($content[hash('md5', $url2)]); + $this->assertArrayHasKey(hash('sha1', $url1), $content); + $this->assertArrayHasKey(hash('sha1', $url2), $content); + $this->assertTrue($content[hash('sha1', $url1)]); + $this->assertFalse($content[hash('sha1', $url2)]); } public function testGetEntriesExistsWhichDoesNotExists() { - $this->client->request('GET', '/api/entries/exists?hashed_url=' . hash('md5', 'http://google.com/entry2')); + $this->client->request('GET', '/api/entries/exists?hashed_url=' . hash('sha1', 'http://google.com/entry2')); $this->assertSame(200, $this->client->getResponse()->getStatusCode()); diff --git a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php index cc1e3fbc..17eed210 100644 --- a/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php +++ b/tests/Wallabag/CoreBundle/Command/GenerateUrlHashesCommandTest.php @@ -72,11 +72,8 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase $entry1->setUrl($url); $em->persist($entry1); - $em->flush(); - $this->assertNull($entry1->getHashedUrl()); - $application = new Application($this->getClient()->getKernel()); $application->add(new GenerateUrlHashesCommand()); @@ -92,7 +89,7 @@ class GenerateUrlHashesCommandTest extends WallabagCoreTestCase $entry = $em->getRepository('WallabagCoreBundle:Entry')->findOneByUrl($url); - $this->assertSame($entry->getHashedUrl(), hash('md5', $url)); + $this->assertSame($entry->getHashedUrl(), hash('sha1', $url)); $query = $em->createQuery('DELETE FROM Wallabag\CoreBundle\Entity\Entry e WHERE e.url = :url'); $query->setParameter('url', $url); -- cgit v1.2.3 From c579ce2306297674c56376a2ab5c8ba66a272253 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 1 Apr 2019 14:34:20 +0200 Subject: Some cleanup Also, do not run the hashed_url migration into a Doctrine migration --- .../ApiBundle/Controller/EntryRestControllerTest.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'tests') diff --git a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php index 34de8ec8..8cc12ed3 100644 --- a/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php +++ b/tests/Wallabag/ApiBundle/Controller/EntryRestControllerTest.php @@ -1076,6 +1076,17 @@ class EntryRestControllerTest extends WallabagApiTestCase } public function testGetEntriesExistsWhichDoesNotExists() + { + $this->client->request('GET', '/api/entries/exists?url=http://google.com/entry2'); + + $this->assertSame(200, $this->client->getResponse()->getStatusCode()); + + $content = json_decode($this->client->getResponse()->getContent(), true); + + $this->assertFalse($content['exists']); + } + + public function testGetEntriesExistsWhichDoesNotExistsWithHashedUrl() { $this->client->request('GET', '/api/entries/exists?hashed_url=' . hash('sha1', 'http://google.com/entry2')); @@ -1087,6 +1098,13 @@ class EntryRestControllerTest extends WallabagApiTestCase } public function testGetEntriesExistsWithNoUrl() + { + $this->client->request('GET', '/api/entries/exists?url='); + + $this->assertSame(403, $this->client->getResponse()->getStatusCode()); + } + + public function testGetEntriesExistsWithNoHashedUrl() { $this->client->request('GET', '/api/entries/exists?hashed_url='); -- cgit v1.2.3