From ae669126e718ede5dbf76929215d8514cd960976 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 12 Jul 2016 13:51:05 +0200 Subject: Import Firefox & Chrome bookmarks into wallabag --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 227 +++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 src/Wallabag/ImportBundle/Import/BrowserImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php new file mode 100644 index 00000000..263a11d5 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -0,0 +1,227 @@ +em = $em; + $this->logger = new NullLogger(); + $this->contentProxy = $contentProxy; + } + + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * We define the user in a custom call because on the import command there is no logged in user. + * So we can't retrieve user from the `security.token_storage` service. + * + * @param User $user + * + * @return $this + */ + public function setUser(User $user) + { + $this->user = $user; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'Firefox & Google Chrome'; + } + + /** + * {@inheritdoc} + */ + public function getUrl() + { + return 'import_browser'; + } + + /** + * {@inheritdoc} + */ + public function getDescription() + { + return 'import.browser.description'; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('WallabagImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('WallabagImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + return false; + } + + $this->nbEntries = 1; + $this->parseEntries($data); + $this->em->flush(); + + return true; + } + + private function parseEntries($data) + { + foreach ($data as $importedEntry) { + $this->parseEntry($importedEntry); + } + $this->totalEntries += count($data); + } + + private function parseEntry($importedEntry) + { + if (!is_array($importedEntry)) { + return; + } + + /* Firefox uses guid while Chrome uses id */ + + if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { + $this->parseEntries($importedEntry); + + return; + } + if (key_exists('children', $importedEntry)) { + $this->parseEntries($importedEntry['children']); + + return; + } + if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { + + /* Firefox uses uri while Chrome uses url */ + + $firefox = key_exists('uri', $importedEntry); + + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { + $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); + ++$this->skippedEntries; + + return; + } + + try { + $entry = $this->contentProxy->updateEntry( + new Entry($this->user), + ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] + ); + } catch (\Exception $e) { + $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); + ++$this->skippedEntries; + + return; + } + + $entry->setArchived($this->markAsRead); + + $this->em->persist($entry); + ++$this->importedEntries; + + // flush every 20 entries + if (($this->nbEntries % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$this->nbEntries; + + /* + + Maybe not useful. Delete at will. + + */ + + $this->logger->info($this->nbEntries.' / '.$this->totalEntries); + } + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + * + * @return $this + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Set file path to the json file. + * + * @param string $filepath + * + * @return $this + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + ]; + } +} -- cgit v1.2.3 From efe659ab84df5db23d95203c0cef8c43ed0914e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 2 Sep 2016 13:53:45 +0200 Subject: Add Chrome path for Mac OS --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 8 -------- 1 file changed, 8 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 263a11d5..e3457196 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -175,14 +175,6 @@ class BrowserImport implements ImportInterface $this->em->clear($entry); } ++$this->nbEntries; - - /* - - Maybe not useful. Delete at will. - - */ - - $this->logger->info($this->nbEntries.' / '.$this->totalEntries); } } -- cgit v1.2.3 From 59201088b4fc13fd361238396f630dabd9bd1990 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 17:47:47 +0200 Subject: bring chrome and firefox as separate imports --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 247 ++++++++++----------- src/Wallabag/ImportBundle/Import/ChromeImport.php | 71 ++++++ src/Wallabag/ImportBundle/Import/FirefoxImport.php | 71 ++++++ 3 files changed, 262 insertions(+), 127 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Import/ChromeImport.php create mode 100644 src/Wallabag/ImportBundle/Import/FirefoxImport.php (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index e3457196..ef7d6d95 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -9,69 +9,24 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; -class BrowserImport implements ImportInterface +abstract class BrowserImport extends AbstractImport { - protected $user; - protected $em; - protected $logger; - protected $contentProxy; - protected $skippedEntries = 0; - protected $importedEntries = 0; - protected $totalEntries = 0; protected $filepath; - protected $markAsRead; - private $nbEntries; - - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->logger = new NullLogger(); - $this->contentProxy = $contentProxy; - } - - public function setLogger(LoggerInterface $logger) - { - $this->logger = $logger; - } - - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - * - * @return $this - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } /** * {@inheritdoc} */ - public function getName() - { - return 'Firefox & Google Chrome'; - } + abstract public function getName(); /** * {@inheritdoc} */ - public function getUrl() - { - return 'import_browser'; - } + abstract public function getUrl(); /** * {@inheritdoc} */ - public function getDescription() - { - return 'import.browser.description'; - } + abstract public function getDescription(); /** * {@inheritdoc} @@ -96,124 +51,162 @@ class BrowserImport implements ImportInterface return false; } - $this->nbEntries = 1; + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); - $this->em->flush(); return true; } - private function parseEntries($data) + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) { - foreach ($data as $importedEntry) { - $this->parseEntry($importedEntry); + $i = 1; + + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } + ++$i; } - $this->totalEntries += count($data); + + $this->em->flush(); } - private function parseEntry($importedEntry) + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) { - if (!is_array($importedEntry)) { - return; + foreach ($entries as $importedEntry) { + + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); } + } - /* Firefox uses guid while Chrome uses id */ + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); - return; } + if (key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); - return; } - if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { - - /* Firefox uses uri while Chrome uses url */ - $firefox = key_exists('uri', $importedEntry); - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; + if (!key_exists('uri', $importedEntry) && !key_exists('url', $importedEntry)) { + return; + } - return; - } + $firefox = key_exists('uri', $importedEntry); - if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { - $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); - ++$this->skippedEntries; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - return; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - try { - $entry = $this->contentProxy->updateEntry( - new Entry($this->user), - ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] - ); - } catch (\Exception $e) { - $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); - ++$this->skippedEntries; + return; + } - return; - } + $data = $this->prepareEntry($importedEntry); - $entry->setArchived($this->markAsRead); + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - $this->em->persist($entry); - ++$this->importedEntries; + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); - // flush every 20 entries - if (($this->nbEntries % 20) === 0) { - $this->em->flush(); - $this->em->clear($entry); - } - ++$this->nbEntries; + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] + ); } - } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - * - * @return $this - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; + $entry->setArchived($data['is_archived']); - return $this; - } + if (!empty($data['created_at'])) { + $dt = new \DateTime(); + $entry->setCreatedAt($dt->setTimestamp($data['created_at']/1000)); + } - /** - * Set file path to the json file. - * - * @param string $filepath - * - * @return $this - */ - public function setFilepath($filepath) - { - $this->filepath = $filepath; + $this->em->persist($entry); + ++$this->importedEntries; - return $this; + return $entry; } /** * {@inheritdoc} */ - public function getSummary() + protected function setEntryAsRead(array $importedEntry) { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; + $importedEntry['is_archived'] = 1; + + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php new file mode 100644 index 00000000..7936ee2f --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php new file mode 100644 index 00000000..cbf10b87 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} -- cgit v1.2.3 From 2c61db30b737685ae9102ec10f2371778fb13f1a Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 18:00:08 +0200 Subject: cs & fixes --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 9 +++------ src/Wallabag/ImportBundle/Import/ChromeImport.php | 10 +--------- src/Wallabag/ImportBundle/Import/FirefoxImport.php | 20 ++++++-------------- 3 files changed, 10 insertions(+), 29 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index ef7d6d95..3e1cb12b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -2,9 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; @@ -121,7 +118,6 @@ abstract class BrowserImport extends AbstractImport protected function parseEntriesForProducer(array $entries) { foreach ($entries as $importedEntry) { - if ((array) $importedEntry !== $importedEntry) { continue; } @@ -144,14 +140,15 @@ abstract class BrowserImport extends AbstractImport */ public function parseEntry(array $importedEntry) { - if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); + return; } if (key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); + return; } @@ -191,7 +188,7 @@ abstract class BrowserImport extends AbstractImport if (!empty($data['created_at'])) { $dt = new \DateTime(); - $entry->setCreatedAt($dt->setTimestamp($data['created_at']/1000)); + $entry->setCreatedAt($dt->setTimestamp($data['created_at'] / 1000)); } $this->em->persist($entry); diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 7936ee2f..941d68d3 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -2,13 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; -use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; -use Wallabag\CoreBundle\Helper\ContentProxy; - class ChromeImport extends BrowserImport { protected $filepath; @@ -37,7 +30,7 @@ class ChromeImport extends BrowserImport return 'import.chrome.description'; } - /** + /** * {@inheritdoc} */ protected function prepareEntry($entry = []) @@ -58,7 +51,6 @@ class ChromeImport extends BrowserImport return $data; } - /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index cbf10b87..351cbef1 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -2,13 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; -use Wallabag\CoreBundle\Entity\Entry; -use Wallabag\UserBundle\Entity\User; -use Wallabag\CoreBundle\Helper\ContentProxy; - class FirefoxImport extends BrowserImport { protected $filepath; @@ -42,7 +35,7 @@ class FirefoxImport extends BrowserImport */ protected function prepareEntry($entry = []) { - $data = [ + $data = [ 'title' => $entry['name'], 'html' => '', 'url' => $entry['url'], @@ -51,13 +44,12 @@ class FirefoxImport extends BrowserImport 'created_at' => $entry['date_added'], ]; - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + return $data; + } /** * {@inheritdoc} -- cgit v1.2.3 From 64b1229b2d711e6b2f0e60de482802d9e86b912f Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 19:24:19 +0200 Subject: fix tests --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 4 +-- src/Wallabag/ImportBundle/Import/ChromeImport.php | 40 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 3e1cb12b..198e148e 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -31,13 +31,13 @@ abstract class BrowserImport extends AbstractImport public function import() { if (!$this->user) { - $this->logger->error('WallabagImport: user is not defined'); + $this->logger->error('Wallabag Browser Import: user is not defined'); return false; } if (!file_exists($this->filepath) || !is_readable($this->filepath)) { - $this->logger->error('WallabagImport: unable to read file', ['filepath' => $this->filepath]); + $this->logger->error('Wallabag Browser Import: unable to read file', ['filepath' => $this->filepath]); return false; } diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 941d68d3..1af7cc87 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -30,26 +30,26 @@ class ChromeImport extends BrowserImport return 'import.chrome.description'; } - /** - * {@inheritdoc} - */ - protected function prepareEntry($entry = []) - { - $data = [ - 'title' => $entry['name'], - 'html' => '', - 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $entry['date_added'], - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } + /** + * {@inheritdoc} + */ + protected function prepareEntry($entry = []) + { + $data = [ + 'title' => $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } /** * {@inheritdoc} -- cgit v1.2.3 From bd206a84d8c333c1287f5cb81d2e29f3afe515c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Fri, 23 Sep 2016 10:56:08 +0200 Subject: Fixed tests by removing clear() --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 198e148e..44315e8b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -94,10 +94,6 @@ abstract class BrowserImport extends AbstractImport // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); - - // clear only affected entities - $this->em->clear(Entry::class); - $this->em->clear(Tag::class); } ++$i; } -- cgit v1.2.3 From 27acc6ddb8b0a1549a3f015171621e3056ef65d2 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 25 Sep 2016 15:29:40 +0200 Subject: Fix bad date format in Browser import --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 23 ++++++++++++++++ src/Wallabag/ImportBundle/Import/ChromeImport.php | 31 ---------------------- src/Wallabag/ImportBundle/Import/FirefoxImport.php | 31 ---------------------- 3 files changed, 23 insertions(+), 62 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 44315e8b..da69df9b 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -193,6 +193,29 @@ abstract class BrowserImport extends AbstractImport return $entry; } + /** + * {@inheritdoc} + */ + protected function prepareEntry($entry = []) + { + $data = [ + 'title' => $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + // date are in format like "13118829474385693" + // and it'll be devided by 1000 in AbstractImport + 'created_at' => (int) ceil($entry['date_added'] / 10000), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 1af7cc87..60602a1b 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -29,35 +29,4 @@ class ChromeImport extends BrowserImport { return 'import.chrome.description'; } - - /** - * {@inheritdoc} - */ - protected function prepareEntry($entry = []) - { - $data = [ - 'title' => $entry['name'], - 'html' => '', - 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $entry['date_added'], - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } - - /** - * {@inheritdoc} - */ - protected function setEntryAsRead(array $importedEntry) - { - $importedEntry['is_archived'] = 1; - - return $importedEntry; - } } diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index 351cbef1..1a0b1154 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -29,35 +29,4 @@ class FirefoxImport extends BrowserImport { return 'import.firefox.description'; } - - /** - * {@inheritdoc} - */ - protected function prepareEntry($entry = []) - { - $data = [ - 'title' => $entry['name'], - 'html' => '', - 'url' => $entry['url'], - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $entry['date_added'], - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } - - /** - * {@inheritdoc} - */ - protected function setEntryAsRead(array $importedEntry) - { - $importedEntry['is_archived'] = 1; - - return $importedEntry; - } } -- cgit v1.2.3 From 12d93e6896f2d99b6329b7979ee7b6d11e457c3a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 25 Sep 2016 22:24:07 +0200 Subject: Update Firefox file With real data, the previous looks more than a Chrome converted file. Also, fix date conversion (hope so). --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 36 ++++++++++++++-------- 1 file changed, 24 insertions(+), 12 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index da69df9b..68fa8bf8 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -136,27 +136,27 @@ abstract class BrowserImport extends AbstractImport */ public function parseEntry(array $importedEntry) { - if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { + if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); return; } - if (key_exists('children', $importedEntry)) { + if (array_key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); return; } - if (!key_exists('uri', $importedEntry) && !key_exists('url', $importedEntry)) { + if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) { return; } - $firefox = key_exists('uri', $importedEntry); + $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url']; $existingEntry = $this->em ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); + ->findByUrlAndUserId($url, $this->user->getId()); if (false !== $existingEntry) { ++$this->skippedEntries; @@ -184,7 +184,7 @@ abstract class BrowserImport extends AbstractImport if (!empty($data['created_at'])) { $dt = new \DateTime(); - $entry->setCreatedAt($dt->setTimestamp($data['created_at'] / 1000)); + $entry->setCreatedAt($dt->setTimestamp($data['created_at'])); } $this->em->persist($entry); @@ -196,17 +196,29 @@ abstract class BrowserImport extends AbstractImport /** * {@inheritdoc} */ - protected function prepareEntry($entry = []) + protected function prepareEntry(array $entry = []) { + $url = array_key_exists('uri', $entry) ? $entry['uri'] : $entry['url']; + $date = array_key_exists('date_added', $entry) ? $entry['date_added'] : $entry['dateAdded']; + $title = array_key_exists('name', $entry) ? $entry['name'] : $entry['title']; + + if (16 === strlen($date)) { + // firefox ... + $date = (int) ceil($date / 1000000); + } else if (17 === strlen($date)) { + // chrome ... + $date = (int) ceil($date / 10000000); + } else { + $date = ''; + } + $data = [ - 'title' => $entry['name'], + 'title' => $title, 'html' => '', - 'url' => $entry['url'], + 'url' => $url, 'is_archived' => $this->markAsRead, 'tags' => '', - // date are in format like "13118829474385693" - // and it'll be devided by 1000 in AbstractImport - 'created_at' => (int) ceil($entry['date_added'] / 10000), + 'created_at' => $date, ]; if (array_key_exists('tags', $entry) && $entry['tags'] != '') { -- cgit v1.2.3 From 990adfb34c148e7cd28b9cb784cf2b7388caae8f Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 26 Sep 2016 07:30:02 +0200 Subject: Move prepareEntry to dedicated place Yeah first try was ugly, now each part are in the dedicated place. Also, the date is hardly truncated to 10 chars because Firefox date are 16 chars long and Chrome are 17 chars long. So instead of divised them by a huge number, I prefer to truncate them. --- src/Wallabag/ImportBundle/Import/BrowserImport.php | 35 ---------------------- src/Wallabag/ImportBundle/Import/ChromeImport.php | 21 +++++++++++++ src/Wallabag/ImportBundle/Import/FirefoxImport.php | 21 +++++++++++++ 3 files changed, 42 insertions(+), 35 deletions(-) (limited to 'src/Wallabag/ImportBundle/Import') diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 68fa8bf8..e15443c4 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -193,41 +193,6 @@ abstract class BrowserImport extends AbstractImport return $entry; } - /** - * {@inheritdoc} - */ - protected function prepareEntry(array $entry = []) - { - $url = array_key_exists('uri', $entry) ? $entry['uri'] : $entry['url']; - $date = array_key_exists('date_added', $entry) ? $entry['date_added'] : $entry['dateAdded']; - $title = array_key_exists('name', $entry) ? $entry['name'] : $entry['title']; - - if (16 === strlen($date)) { - // firefox ... - $date = (int) ceil($date / 1000000); - } else if (17 === strlen($date)) { - // chrome ... - $date = (int) ceil($date / 10000000); - } else { - $date = ''; - } - - $data = [ - 'title' => $title, - 'html' => '', - 'url' => $url, - 'is_archived' => $this->markAsRead, - 'tags' => '', - 'created_at' => $date, - ]; - - if (array_key_exists('tags', $entry) && $entry['tags'] != '') { - $data['tags'] = $entry['tags']; - } - - return $data; - } - /** * {@inheritdoc} */ diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php index 60602a1b..d7620bcb 100644 --- a/src/Wallabag/ImportBundle/Import/ChromeImport.php +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -29,4 +29,25 @@ class ChromeImport extends BrowserImport { return 'import.chrome.description'; } + + /** + * {@inheritdoc} + */ + protected function prepareEntry(array $entry = []) + { + $data = [ + 'title' => $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => substr($entry['date_added'], 0, 10), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } } diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php index 1a0b1154..e010f5a4 100644 --- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -29,4 +29,25 @@ class FirefoxImport extends BrowserImport { return 'import.firefox.description'; } + + /** + * {@inheritdoc} + */ + protected function prepareEntry(array $entry = []) + { + $data = [ + 'title' => $entry['title'], + 'html' => '', + 'url' => $entry['uri'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => substr($entry['dateAdded'], 0, 10), + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } } -- cgit v1.2.3