From 03e3753f6bd36f12c0757c76b49b683c49de48ae Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Thu, 1 Sep 2016 08:00:30 +0200 Subject: [PATCH] Add Readability import Based on the JSON export instead of the API (which will be shutting down by the September 30, 2016) --- .../Resources/translations/messages.en.yml | 4 + .../Controller/ReadabilityController.php | 65 +++++++ .../ImportBundle/Import/ReadabilityImport.php | 181 ++++++++++++++++++ .../Resources/config/services.yml | 10 + .../views/Readability/index.html.twig | 43 +++++ .../ImportBundle/fixtures/readability.json | 25 +++ 6 files changed, 328 insertions(+) create mode 100644 src/Wallabag/ImportBundle/Controller/ReadabilityController.php create mode 100644 src/Wallabag/ImportBundle/Import/ReadabilityImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig create mode 100644 tests/Wallabag/ImportBundle/fixtures/readability.json diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 220c4d9c..f2bf9aed 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -341,6 +341,10 @@ import: wallabag_v2: page_title: 'Import > Wallabag v2' description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.' + readability: + page_title: 'Import > Readability' + description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact)' + how_to: 'Please select your Readability export and click on the below button to upload and import it.' developer: page_title: 'Developer' diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php new file mode 100644 index 00000000..b61aa99c --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php @@ -0,0 +1,65 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $readability = $this->get('wallabag_import.readability.import'); + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = 'readability_'.$this->getUser()->getId().'.json'; + + if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $readability + ->setUser($this->getUser()) + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $readability->getSummary(); + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render('WallabagImportBundle:Readability:index.html.twig', [ + 'form' => $form->createView(), + 'import' => $readability, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php new file mode 100644 index 00000000..abea81a7 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php @@ -0,0 +1,181 @@ +user = $user; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function getName() + { + return 'Readability'; + } + + /** + * {@inheritdoc} + */ + public function getUrl() + { + return 'import_readability'; + } + + /** + * {@inheritdoc} + */ + public function getDescription() + { + return 'import.readability.description'; + } + + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Set whether articles must be all marked as read. + * + * @param bool $markAsRead + */ + public function setMarkAsRead($markAsRead) + { + $this->markAsRead = $markAsRead; + + return $this; + } + + /** + * Get whether articles must be all marked as read. + */ + public function getMarkAsRead() + { + return $this->markAsRead; + } + + /** + * {@inheritdoc} + */ + public function getSummary() + { + return [ + 'skipped' => $this->skippedEntries, + 'imported' => $this->importedEntries, + ]; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('ReadabilityImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data) || empty($data['bookmarks'])) { + return false; + } + + $this->parseEntries($data['bookmarks']); + + return true; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) + { + $i = 1; + + foreach ($entries as $importedEntry) { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + continue; + } + + $data = [ + 'title' => $importedEntry['article__title'], + // 'html' => $importedEntry['article__excerpt'], + 'url' => $importedEntry['article__url'], + 'content_type' => '', + 'language' => '', + 'is_archived' => $importedEntry['archive'] || $this->markAsRead, + 'is_starred' => $importedEntry['favorite'], + ]; + + $entry = $this->fetchContent( + new Entry($this->user), + $data['url'], + $data + ); + + // jump to next entry in case of problem while getting content + if (false === $entry) { + ++$this->skippedEntries; + continue; + } + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + + $this->em->persist($entry); + ++$this->importedEntries; + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + $this->em->clear($entry); + } + ++$i; + } + + $this->em->flush(); + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 86b44cb3..520d43af 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -43,3 +43,13 @@ services: - [ setLogger, [ "@logger" ]] tags: - { name: wallabag_import.import, alias: wallabag_v2 } + + wallabag_import.readability.import: + class: Wallabag\ImportBundle\Import\ReadabilityImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: readability } diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig new file mode 100644 index 00000000..f527d309 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.readability.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans }}
+

{{ 'import.readability.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Wallabag/ImportBundle/fixtures/readability.json b/tests/Wallabag/ImportBundle/fixtures/readability.json new file mode 100644 index 00000000..34379905 --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/readability.json @@ -0,0 +1,25 @@ +{ + "bookmarks": [ + { + "article__excerpt": "When Twitter started it had so much promise to change the way we communicate. But now it has been ruined by the amount of garbage and hate we have to wade through. It’s like that polluted…", + "favorite": false, + "date_archived": null, + "article__url": "https://venngage.com/blog/hashtags-are-worthless/", + "date_added": "2016-08-25T12:05:00", + "date_favorited": null, + "article__title": "We Looked At 167,943 Tweets & Found Out Hashtags Are Worthless", + "archive": false + }, + { + "article__excerpt": "TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data. Here’s a demo and the code for the infinite…", + "favorite": false, + "date_archived": "2016-08-26T12:21:54", + "article__url": "https://developers.google.com/web/updates/2016/07/infinite-scroller?imm_mid=0e6839&cmp=em-webops-na-na-newsltr_20160805", + "date_added": "2016-08-06T05:35:26", + "date_favorited": null, + "article__title": "Complexities of an infinite scroller | Web Updates", + "archive": true + } + ], + "recommendations": [] +} -- 2.41.0