From ae669126e718ede5dbf76929215d8514cd960976 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 12 Jul 2016 13:51:05 +0200 Subject: Import Firefox & Chrome bookmarks into wallabag --- .../ImportBundle/Command/ImportCommand.php | 19 +- .../ImportBundle/Controller/BrowserController.php | 91 +++++++++ src/Wallabag/ImportBundle/Import/BrowserImport.php | 227 +++++++++++++++++++++ .../ImportBundle/Resources/config/services.yml | 10 + .../Resources/views/Browser/index.html.twig | 43 ++++ .../Resources/views/Import/index.html.twig | 2 +- 6 files changed, 386 insertions(+), 6 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/BrowserController.php create mode 100644 src/Wallabag/ImportBundle/Import/BrowserImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig (limited to 'src/Wallabag/ImportBundle') diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 20ecc6e1..537dffc2 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -17,7 +17,7 @@ class ImportCommand extends ContainerAwareCommand ->setDescription('Import entries from a JSON export from a wallabag v1 instance') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1 or v2', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2 or browser', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -40,10 +40,19 @@ class ImportCommand extends ContainerAwareCommand throw new Exception(sprintf('User with id "%s" not found', $input->getArgument('userId'))); } - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); - - if ('v2' === $input->getOption('importer')) { - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + switch ($input->getOption('importer')) { + case 'v2': + $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + break; + case 'v1': + $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); + break; + case 'browser': + $wallabag = $this->getContainer()->get('wallabag_import.browser.import'); + break; + default: + $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); + break; } $wallabag->setMarkAsRead($input->getOption('markAsRead')); diff --git a/src/Wallabag/ImportBundle/Controller/BrowserController.php b/src/Wallabag/ImportBundle/Controller/BrowserController.php new file mode 100644 index 00000000..3b54a72e --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/BrowserController.php @@ -0,0 +1,91 @@ +get('wallabag_import.browser.import'); + } + + /** + * Return the template used for the form. + * + * @return string + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Browser:index.html.twig'; + } + + /** + * @Route("/browser", name="import_browser") + * + * @param Request $request + * + * @return Response + */ + public function indexAction(Request $request) + { + $form = $this->createForm(UploadImportType::class); + $form->handleRequest($request); + + $wallabag = $this->getImportService(); + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = $this->getUser()->getId().'.json'; + + if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $wallabag + ->setUser($this->getUser()) + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $wallabag->getSummary(); + // TODO : Pluralize these messages + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render($this->getImportTemplate(), [ + 'form' => $form->createView(), + 'import' => $wallabag, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php new file mode 100644 index 00000000..263a11d5 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -0,0 +1,227 @@ +em = $em; + $this->logger = new NullLogger(); + $this->contentProxy = $contentProxy; + } + + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * We define the user in a custom call because on the import command there is no logged in user. + * So we can't retrieve user from the `security.token_storage` service. + * + * @param User $user + * + * @return $this + */ + public function setUser(User $user) + { + $this->user = $user; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'Firefox & Google Chrome'; + } + + /** + * {@inheritdoc} + */ + public function getUrl() + { + return 'import_browser'; + } + + /** + * {@inheritdoc} + */ + public function getDescription() + { + return 'import.browser.description'; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('WallabagImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('WallabagImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + return false; + } + + $this->nbEntries = 1; + $this->parseEntries($data); + $this->em->flush(); + + return true; + } + + private function parseEntries($data) + { + foreach ($data as $importedEntry) { + $this->parseEntry($importedEntry); + } + $this->totalEntries += count($data); + } + + private function parseEntry($importedEntry) + { + if (!is_array($importedEntry)) { + return; + } + + /* Firefox uses guid while Chrome uses id */ + + if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { + $this->parseEntries($importedEntry); + + return; + } + if (key_exists('children', $importedEntry)) { + $this->parseEntries($importedEntry['children']); + + return; + } + if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { + + /* Firefox uses uri while Chrome uses url */ + + $firefox = key_exists('uri', $importedEntry); + + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { + $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); + ++$this->skippedEntries; + + return; + } + + try { + $entry = $this->contentProxy->updateEntry( + new Entry($this->user), + ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] + ); + } catch (\Exception $e) { + $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); + ++$this->skippedEntries; + + return; + } + + $entry->setArchived($this->markAsRead); + + $this->em->persist($entry); + ++$this->importedEntries; + + // flush every 20 entries + if (($this->nbEntries % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$this->nbEntries; + + /* + + Maybe not useful. Delete at will. + + */ + + $this->logger->info($this->nbEntries.' / '.$this->totalEntries); + } + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + * + * @return $this + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Set file path to the json file. + * + * @param string $filepath + * + * @return $this + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + ]; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index f03404ae..d8be5c28 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -56,3 +56,13 @@ services: - [ setLogger, [ "@logger" ]] tags: - { name: wallabag_import.import, alias: readability } + + wallabag_import.browser.import: + class: Wallabag\ImportBundle\Import\BrowserImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: browser } diff --git a/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig new file mode 100644 index 00000000..bfc74e9d --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.browser.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans|raw }}
+

{{ 'import.browser.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index aebbfa20..6ea5e0f4 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig @@ -11,7 +11,7 @@ {% for import in imports %}
  • {{ import.name }}
    -
    {{ import.description|trans }}
    +
    {{ import.description|trans|raw }}

    {{ 'import.action.import_contents'|trans }}

    • {% endfor %} -- cgit v1.2.3