From a973afeac7b7399d35b881920f0afc1947765ccd Mon Sep 17 00:00:00 2001 From: VirtualTam Date: Thu, 28 Jul 2016 22:54:33 +0200 Subject: Refactor bookmark import using a generic Netscape parser Relates to #607 Relates to #608 Relates to #493 (abandoned) Additions: - use Composer's autoload to load 3rd-party dependencies under vendor/ Modifications: - [import] replace the current parser with a generic, stable parser - move code to application/NetscapeBookmarkUtils - improve status report after parsing - [router] use the same endpoint for both bookmark upload and import dialog - [template] update bookmark import options - allow adding tags to all imported links - allow selecting the visibility (privacy) of imported links - [tests] ensure bookmarks are properly parsed and imported in the LinkDB - reuse reference input from the parser's test data See: - https://github.com/shaarli/netscape-bookmark-parser - https://getcomposer.org/doc/01-basic-usage.md#autoloading Signed-off-by: VirtualTam --- tests/NetscapeBookmarkUtils/BookmarkExportTest.php | 134 ++++++ tests/NetscapeBookmarkUtils/BookmarkImportTest.php | 518 +++++++++++++++++++++ tests/NetscapeBookmarkUtils/input/empty.htm | 0 .../NetscapeBookmarkUtils/input/netscape_basic.htm | 11 + .../input/netscape_nested.htm | 31 ++ tests/NetscapeBookmarkUtils/input/no_doctype.htm | 7 + tests/NetscapeBookmarkUtils/input/same_date.htm | 11 + 7 files changed, 712 insertions(+) create mode 100644 tests/NetscapeBookmarkUtils/BookmarkExportTest.php create mode 100644 tests/NetscapeBookmarkUtils/BookmarkImportTest.php create mode 100644 tests/NetscapeBookmarkUtils/input/empty.htm create mode 100644 tests/NetscapeBookmarkUtils/input/netscape_basic.htm create mode 100644 tests/NetscapeBookmarkUtils/input/netscape_nested.htm create mode 100644 tests/NetscapeBookmarkUtils/input/no_doctype.htm create mode 100644 tests/NetscapeBookmarkUtils/input/same_date.htm (limited to 'tests/NetscapeBookmarkUtils') diff --git a/tests/NetscapeBookmarkUtils/BookmarkExportTest.php b/tests/NetscapeBookmarkUtils/BookmarkExportTest.php new file mode 100644 index 00000000..cc54ab9f --- /dev/null +++ b/tests/NetscapeBookmarkUtils/BookmarkExportTest.php @@ -0,0 +1,134 @@ +write(self::$testDatastore); + self::$linkDb = new LinkDB(self::$testDatastore, true, false); + } + + /** + * Attempt to export an invalid link selection + * @expectedException Exception + * @expectedExceptionMessageRegExp /Invalid export selection/ + */ + public function testFilterAndFormatInvalid() + { + NetscapeBookmarkUtils::filterAndFormat(self::$linkDb, 'derp', false, ''); + } + + /** + * Prepare all links for export + */ + public function testFilterAndFormatAll() + { + $links = NetscapeBookmarkUtils::filterAndFormat(self::$linkDb, 'all', false, ''); + $this->assertEquals(self::$refDb->countLinks(), sizeof($links)); + foreach ($links as $link) { + $date = DateTime::createFromFormat(LinkDB::LINK_DATE_FORMAT, $link['linkdate']); + $this->assertEquals( + $date->getTimestamp(), + $link['timestamp'] + ); + $this->assertEquals( + str_replace(' ', ',', $link['tags']), + $link['taglist'] + ); + } + } + + /** + * Prepare private links for export + */ + public function testFilterAndFormatPrivate() + { + $links = NetscapeBookmarkUtils::filterAndFormat(self::$linkDb, 'private', false, ''); + $this->assertEquals(self::$refDb->countPrivateLinks(), sizeof($links)); + foreach ($links as $link) { + $date = DateTime::createFromFormat(LinkDB::LINK_DATE_FORMAT, $link['linkdate']); + $this->assertEquals( + $date->getTimestamp(), + $link['timestamp'] + ); + $this->assertEquals( + str_replace(' ', ',', $link['tags']), + $link['taglist'] + ); + } + } + + /** + * Prepare public links for export + */ + public function testFilterAndFormatPublic() + { + $links = NetscapeBookmarkUtils::filterAndFormat(self::$linkDb, 'public', false, ''); + $this->assertEquals(self::$refDb->countPublicLinks(), sizeof($links)); + foreach ($links as $link) { + $date = DateTime::createFromFormat(LinkDB::LINK_DATE_FORMAT, $link['linkdate']); + $this->assertEquals( + $date->getTimestamp(), + $link['timestamp'] + ); + $this->assertEquals( + str_replace(' ', ',', $link['tags']), + $link['taglist'] + ); + } + } + + /** + * Do not prepend notes with the Shaarli index's URL + */ + public function testFilterAndFormatDoNotPrependNoteUrl() + { + $links = NetscapeBookmarkUtils::filterAndFormat(self::$linkDb, 'public', false, ''); + $this->assertEquals( + '?WDWyig', + $links[0]['url'] + ); + } + + /** + * Prepend notes with the Shaarli index's URL + */ + public function testFilterAndFormatPrependNoteUrl() + { + $indexUrl = 'http://localhost:7469/shaarli/'; + $links = NetscapeBookmarkUtils::filterAndFormat( + self::$linkDb, + 'public', + true, + $indexUrl + ); + $this->assertEquals( + $indexUrl . '?WDWyig', + $links[0]['url'] + ); + } +} diff --git a/tests/NetscapeBookmarkUtils/BookmarkImportTest.php b/tests/NetscapeBookmarkUtils/BookmarkImportTest.php new file mode 100644 index 00000000..2d4e7557 --- /dev/null +++ b/tests/NetscapeBookmarkUtils/BookmarkImportTest.php @@ -0,0 +1,518 @@ + array( + 'name' => $filename, + 'tmp_name' => __DIR__ . '/input/' . $filename, + 'size' => filesize(__DIR__ . '/input/' . $filename) + ) + ); +} + + +/** + * Netscape bookmark import + */ +class BookmarkImportTest extends PHPUnit_Framework_TestCase +{ + /** + * @var string datastore to test write operations + */ + protected static $testDatastore = 'sandbox/datastore.php'; + + /** + * @var LinkDB private LinkDB instance + */ + protected $linkDb = null; + + /** + * @var string Dummy page cache + */ + protected $pagecache = 'tests'; + + /** + * Resets test data before each test + */ + protected function setUp() + { + if (file_exists(self::$testDatastore)) { + unlink(self::$testDatastore); + } + // start with an empty datastore + file_put_contents(self::$testDatastore, ''); + $this->linkDb = new LinkDB(self::$testDatastore, true, false); + } + + /** + * Attempt to import bookmarks from an empty file + */ + public function testImportEmptyData() + { + $files = file2array('empty.htm'); + $this->assertEquals( + 'File empty.htm (0 bytes) has an unknown file format.' + .' Nothing was imported.', + NetscapeBookmarkUtils::import(NULL, $files, NULL, NULL) + ); + $this->assertEquals(0, count($this->linkDb)); + } + + /** + * Attempt to import bookmarks from a file with no Doctype + */ + public function testImportNoDoctype() + { + $files = file2array('no_doctype.htm'); + $this->assertEquals( + 'File no_doctype.htm (350 bytes) has an unknown file format. Nothing was imported.', + NetscapeBookmarkUtils::import(NULL, $files, NULL, NULL) + ); + $this->assertEquals(0, count($this->linkDb)); + } + + /** + * Import bookmarks nested in a folder hierarchy + */ + public function testImportNested() + { + $files = file2array('netscape_nested.htm'); + $this->assertEquals( + 'File netscape_nested.htm (1337 bytes) was successfully processed:' + .' 8 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(8, count($this->linkDb)); + $this->assertEquals(2, count_private($this->linkDb)); + + $this->assertEquals( + array( + 'linkdate' => '20160225_205541', + 'title' => 'Nested 1', + 'url' => 'http://nest.ed/1', + 'description' => '', + 'private' => 0, + 'tags' => 'tag1 tag2' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/1') + ); + $this->assertEquals( + array( + 'linkdate' => '20160225_205542', + 'title' => 'Nested 1-1', + 'url' => 'http://nest.ed/1-1', + 'description' => '', + 'private' => 0, + 'tags' => 'folder1 tag1 tag2' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/1-1') + ); + $this->assertEquals( + array( + 'linkdate' => '20160225_205547', + 'title' => 'Nested 1-2', + 'url' => 'http://nest.ed/1-2', + 'description' => '', + 'private' => 0, + 'tags' => 'folder1 tag3 tag4' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/1-2') + ); + $this->assertEquals( + array( + 'linkdate' => '20160202_172222', + 'title' => 'Nested 2-1', + 'url' => 'http://nest.ed/2-1', + 'description' => 'First link of the second section', + 'private' => 1, + 'tags' => 'folder2' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/2-1') + ); + $this->assertEquals( + array( + 'linkdate' => '20160119_200227', + 'title' => 'Nested 2-2', + 'url' => 'http://nest.ed/2-2', + 'description' => 'Second link of the second section', + 'private' => 1, + 'tags' => 'folder2' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/2-2') + ); + $this->assertEquals( + array( + 'linkdate' => '20160202_172223', + 'title' => 'Nested 3-1', + 'url' => 'http://nest.ed/3-1', + 'description' => '', + 'private' => 0, + 'tags' => 'folder3 folder3-1 tag3' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/3-1') + ); + $this->assertEquals( + array( + 'linkdate' => '20160119_200228', + 'title' => 'Nested 3-2', + 'url' => 'http://nest.ed/3-2', + 'description' => '', + 'private' => 0, + 'tags' => 'folder3 folder3-1' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/3-2') + ); + $this->assertEquals( + array( + 'linkdate' => '20160229_081541', + 'title' => 'Nested 2', + 'url' => 'http://nest.ed/2', + 'description' => '', + 'private' => 0, + 'tags' => 'tag4' + ), + $this->linkDb->getLinkFromUrl('http://nest.ed/2') + ); + } + + /** + * Import bookmarks with the default privacy setting (reuse from file) + * + * The $_POST array is not set. + */ + public function testImportDefaultPrivacyNoPost() + { + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(1, count_private($this->linkDb)); + + $this->assertEquals( + array( + 'linkdate' => '20001010_105536', + 'title' => 'Secret stuff', + 'url' => 'https://private.tld', + 'description' => "Super-secret stuff you're not supposed to know about", + 'private' => 1, + 'tags' => 'private secret' + ), + $this->linkDb->getLinkFromUrl('https://private.tld') + ); + $this->assertEquals( + array( + 'linkdate' => '20160225_205548', + 'title' => 'Public stuff', + 'url' => 'http://public.tld', + 'description' => '', + 'private' => 0, + 'tags' => 'public hello world' + ), + $this->linkDb->getLinkFromUrl('http://public.tld') + ); + } + + /** + * Import bookmarks with the default privacy setting (reuse from file) + */ + public function testImportKeepPrivacy() + { + $post = array('privacy' => 'default'); + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(1, count_private($this->linkDb)); + + $this->assertEquals( + array( + 'linkdate' => '20001010_105536', + 'title' => 'Secret stuff', + 'url' => 'https://private.tld', + 'description' => "Super-secret stuff you're not supposed to know about", + 'private' => 1, + 'tags' => 'private secret' + ), + $this->linkDb->getLinkFromUrl('https://private.tld') + ); + $this->assertEquals( + array( + 'linkdate' => '20160225_205548', + 'title' => 'Public stuff', + 'url' => 'http://public.tld', + 'description' => '', + 'private' => 0, + 'tags' => 'public hello world' + ), + $this->linkDb->getLinkFromUrl('http://public.tld') + ); + } + + /** + * Import links as public + */ + public function testImportAsPublic() + { + $post = array('privacy' => 'public'); + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + $this->assertEquals( + 0, + $this->linkDb['20001010_105536']['private'] + ); + $this->assertEquals( + 0, + $this->linkDb['20160225_205548']['private'] + ); + } + + /** + * Import links as private + */ + public function testImportAsPrivate() + { + $post = array('privacy' => 'private'); + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(2, count_private($this->linkDb)); + $this->assertEquals( + 1, + $this->linkDb['20001010_105536']['private'] + ); + $this->assertEquals( + 1, + $this->linkDb['20160225_205548']['private'] + ); + } + + /** + * Overwrite private links so they become public + */ + public function testOverwriteAsPublic() + { + $files = file2array('netscape_basic.htm'); + + // import links as private + $post = array('privacy' => 'private'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(2, count_private($this->linkDb)); + $this->assertEquals( + 1, + $this->linkDb['20001010_105536']['private'] + ); + $this->assertEquals( + 1, + $this->linkDb['20160225_205548']['private'] + ); + + // re-import as public, enable overwriting + $post = array( + 'privacy' => 'public', + 'overwrite' => 'true' + ); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 2 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + $this->assertEquals( + 0, + $this->linkDb['20001010_105536']['private'] + ); + $this->assertEquals( + 0, + $this->linkDb['20160225_205548']['private'] + ); + } + + /** + * Overwrite public links so they become private + */ + public function testOverwriteAsPrivate() + { + $files = file2array('netscape_basic.htm'); + + // import links as public + $post = array('privacy' => 'public'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + $this->assertEquals( + 0, + $this->linkDb['20001010_105536']['private'] + ); + $this->assertEquals( + 0, + $this->linkDb['20160225_205548']['private'] + ); + + // re-import as private, enable overwriting + $post = array( + 'privacy' => 'private', + 'overwrite' => 'true' + ); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 2 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(2, count_private($this->linkDb)); + $this->assertEquals( + 1, + $this->linkDb['20001010_105536']['private'] + ); + $this->assertEquals( + 1, + $this->linkDb['20160225_205548']['private'] + ); + } + + /** + * Attept to import the same links twice without enabling overwriting + */ + public function testSkipOverwrite() + { + $post = array('privacy' => 'public'); + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + + // re-import as private, DO NOT enable overwriting + $post = array('privacy' => 'private'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 0 links imported, 0 links overwritten, 2 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + } + + /** + * Add user-specified tags to all imported bookmarks + */ + public function testSetDefaultTags() + { + $post = array( + 'privacy' => 'public', + 'default_tags' => 'tag1,tag2 tag3' + ); + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + $this->assertEquals( + 'tag1 tag2 tag3 private secret', + $this->linkDb['20001010_105536']['tags'] + ); + $this->assertEquals( + 'tag1 tag2 tag3 public hello world', + $this->linkDb['20160225_205548']['tags'] + ); + } + + /** + * The user-specified tags contain characters to be escaped + */ + public function testSanitizeDefaultTags() + { + $post = array( + 'privacy' => 'public', + 'default_tags' => 'tag1&,tag2 "tag3"' + ); + $files = file2array('netscape_basic.htm'); + $this->assertEquals( + 'File netscape_basic.htm (482 bytes) was successfully processed:' + .' 2 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import($post, $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(2, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + $this->assertEquals( + 'tag1& tag2 "tag3" private secret', + $this->linkDb['20001010_105536']['tags'] + ); + $this->assertEquals( + 'tag1& tag2 "tag3" public hello world', + $this->linkDb['20160225_205548']['tags'] + ); + } + + /** + * Ensure each imported bookmark has a unique linkdate + * + * See https://github.com/shaarli/Shaarli/issues/351 + */ + public function testImportSameDate() + { + $files = file2array('same_date.htm'); + $this->assertEquals( + 'File same_date.htm (453 bytes) was successfully processed:' + .' 3 links imported, 0 links overwritten, 0 links skipped.', + NetscapeBookmarkUtils::import(array(), $files, $this->linkDb, $this->pagecache) + ); + $this->assertEquals(3, count($this->linkDb)); + $this->assertEquals(0, count_private($this->linkDb)); + $this->assertEquals( + '20160225_205548', + $this->linkDb['20160225_205548']['linkdate'] + ); + $this->assertEquals( + '20160225_205549', + $this->linkDb['20160225_205549']['linkdate'] + ); + $this->assertEquals( + '20160225_205550', + $this->linkDb['20160225_205550']['linkdate'] + ); + } +} diff --git a/tests/NetscapeBookmarkUtils/input/empty.htm b/tests/NetscapeBookmarkUtils/input/empty.htm new file mode 100644 index 00000000..e69de29b diff --git a/tests/NetscapeBookmarkUtils/input/netscape_basic.htm b/tests/NetscapeBookmarkUtils/input/netscape_basic.htm new file mode 100644 index 00000000..affe0cf8 --- /dev/null +++ b/tests/NetscapeBookmarkUtils/input/netscape_basic.htm @@ -0,0 +1,11 @@ + + +Bookmarks +

Bookmarks

+

+

Secret stuff +
Super-secret stuff you're not supposed to know about +
Public stuff +

diff --git a/tests/NetscapeBookmarkUtils/input/netscape_nested.htm b/tests/NetscapeBookmarkUtils/input/netscape_nested.htm new file mode 100644 index 00000000..b486fe18 --- /dev/null +++ b/tests/NetscapeBookmarkUtils/input/netscape_nested.htm @@ -0,0 +1,31 @@ + + +Bookmarks +

Bookmarks

+

+

Nested 1 +

Folder1

+

+

Nested 1-1 +
Nested 1-2 +

+

Folder2

+
This second folder contains wonderful links! +

+

Nested 2-1 +
First link of the second section +
Nested 2-2 +
Second link of the second section +

+

Folder3

+

+

Folder3-1

+

+

Nested 3-1 +
Nested 3-2 +

+

+

Nested 2 +

diff --git a/tests/NetscapeBookmarkUtils/input/no_doctype.htm b/tests/NetscapeBookmarkUtils/input/no_doctype.htm new file mode 100644 index 00000000..766d398b --- /dev/null +++ b/tests/NetscapeBookmarkUtils/input/no_doctype.htm @@ -0,0 +1,7 @@ +Bookmarks +

Bookmarks

+

+

Secret stuff +
Super-secret stuff you're not supposed to know about +
Public stuff +

diff --git a/tests/NetscapeBookmarkUtils/input/same_date.htm b/tests/NetscapeBookmarkUtils/input/same_date.htm new file mode 100644 index 00000000..9d58a582 --- /dev/null +++ b/tests/NetscapeBookmarkUtils/input/same_date.htm @@ -0,0 +1,11 @@ + + +Bookmarks +

Bookmarks

+

+

Today's first link +
Today's second link +
Today's third link +

-- cgit v1.2.3