Add Readability import

Based on the JSON export instead of the API (which will be shutting down by the September 30, 2016)

6 files changed, 328 insertions, 0 deletions

diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
index 220c4d9c..f2bf9aed 100644
--- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
+++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
@@ -341,6 +341,10 @@ import:
     wallabag_v2:
         page_title: 'Import > Wallabag v2'
         description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.'
+    readability:
+        page_title: 'Import > Readability'
+        description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact)'
+        how_to: 'Please select your Readability export and click on the below button to upload and import it.'
 
 developer:
     page_title: 'Developer'
diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
new file mode 100644
index 00000000..b61aa99c
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
@@ -0,0 +1,65 @@
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Symfony\Bundle\FrameworkBundle\Controller\Controller;
+use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
+use Symfony\Component\HttpFoundation\Request;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+
+class ReadabilityController extends Controller
+{
+    /**
+     * @Route("/readability", name="import_readability")
+     */
+    public function indexAction(Request $request)
+    {
+        $form = $this->createForm(UploadImportType::class);
+        $form->handleRequest($request);
+
+        $readability = $this->get('wallabag_import.readability.import');
+
+        if ($form->isValid()) {
+            $file = $form->get('file')->getData();
+            $markAsRead = $form->get('mark_as_read')->getData();
+            $name = 'readability_'.$this->getUser()->getId().'.json';
+
+            if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+                $res = $readability
+                    ->setUser($this->getUser())
+                    ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
+                    ->setMarkAsRead($markAsRead)
+                    ->import();
+
+                $message = 'flashes.import.notice.failed';
+
+                if (true === $res) {
+                    $summary = $readability->getSummary();
+                    $message = $this->get('translator')->trans('flashes.import.notice.summary', [
+                        '%imported%' => $summary['imported'],
+                        '%skipped%' => $summary['skipped'],
+                    ]);
+
+                    unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
+                }
+
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    $message
+                );
+
+                return $this->redirect($this->generateUrl('homepage'));
+            } else {
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    'flashes.import.notice.failed_on_file'
+                );
+            }
+        }
+
+        return $this->render('WallabagImportBundle:Readability:index.html.twig', [
+            'form' => $form->createView(),
+            'import' => $readability,
+        ]);
+    }
+}
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
new file mode 100644
index 00000000..abea81a7
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -0,0 +1,181 @@
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\UserBundle\Entity\User;
+
+class ReadabilityImport extends AbstractImport
+{
+    private $user;
+    private $client;
+    private $skippedEntries = 0;
+    private $importedEntries = 0;
+    private $filepath;
+    private $markAsRead;
+
+    /**
+     * We define the user in a custom call because on the import command there is no logged in user.
+     * So we can't retrieve user from the `security.token_storage` service.
+     *
+     * @param User $user
+     */
+    public function setUser(User $user)
+    {
+        $this->user = $user;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getName()
+    {
+        return 'Readability';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getUrl()
+    {
+        return 'import_readability';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDescription()
+    {
+        return 'import.readability.description';
+    }
+
+    /**
+     * Set file path to the json file.
+     *
+     * @param string $filepath
+     */
+    public function setFilepath($filepath)
+    {
+        $this->filepath = $filepath;
+
+        return $this;
+    }
+
+    /**
+     * Set whether articles must be all marked as read.
+     *
+     * @param bool $markAsRead
+     */
+    public function setMarkAsRead($markAsRead)
+    {
+        $this->markAsRead = $markAsRead;
+
+        return $this;
+    }
+
+    /**
+     * Get whether articles must be all marked as read.
+     */
+    public function getMarkAsRead()
+    {
+        return $this->markAsRead;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSummary()
+    {
+        return [
+            'skipped' => $this->skippedEntries,
+            'imported' => $this->importedEntries,
+        ];
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function import()
+    {
+        if (!$this->user) {
+            $this->logger->error('ReadabilityImport: user is not defined');
+
+            return false;
+        }
+
+        if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+            $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]);
+
+            return false;
+        }
+
+        $data = json_decode(file_get_contents($this->filepath), true);
+
+        if (empty($data) || empty($data['bookmarks'])) {
+            return false;
+        }
+
+        $this->parseEntries($data['bookmarks']);
+
+        return true;
+    }
+
+    /**
+     * Parse and insert all given entries.
+     *
+     * @param $entries
+     */
+    protected function parseEntries($entries)
+    {
+        $i = 1;
+
+        foreach ($entries as $importedEntry) {
+            $existingEntry = $this->em
+                ->getRepository('WallabagCoreBundle:Entry')
+                ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
+
+            if (false !== $existingEntry) {
+                ++$this->skippedEntries;
+                continue;
+            }
+
+            $data = [
+                'title' => $importedEntry['article__title'],
+                // 'html' => $importedEntry['article__excerpt'],
+                'url' => $importedEntry['article__url'],
+                'content_type' => '',
+                'language' => '',
+                'is_archived' => $importedEntry['archive'] || $this->markAsRead,
+                'is_starred' => $importedEntry['favorite'],
+            ];
+
+            $entry = $this->fetchContent(
+                new Entry($this->user),
+                $data['url'],
+                $data
+            );
+
+            // jump to next entry in case of problem while getting content
+            if (false === $entry) {
+                ++$this->skippedEntries;
+                continue;
+            }
+            $entry->setArchived($data['is_archived']);
+            $entry->setStarred($data['is_starred']);
+
+            $this->em->persist($entry);
+            ++$this->importedEntries;
+
+            // flush every 20 entries
+            if (($i % 20) === 0) {
+                $this->em->flush();
+                $this->em->clear($entry);
+            }
+            ++$i;
+        }
+
+        $this->em->flush();
+    }
+}
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index 86b44cb3..520d43af 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -43,3 +43,13 @@ services:
             - [ setLogger, [ "@logger" ]]
         tags:
             -  { name: wallabag_import.import, alias: wallabag_v2 }
+
+    wallabag_import.readability.import:
+        class: Wallabag\ImportBundle\Import\ReadabilityImport
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_core.content_proxy"
+        calls:
+            - [ setLogger, [ "@logger" ]]
+        tags:
+            -  { name: wallabag_import.import, alias: readability }
diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
new file mode 100644
index 00000000..f527d309
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
@@ -0,0 +1,43 @@
+{% extends "WallabagCoreBundle::layout.html.twig" %}
+
+{% block title %}{{ 'import.readability.page_title'|trans }}{% endblock %}
+
+{% block content %}
+<div class="row">
+    <div class="col s12">
+        <div class="card-panel settings">
+            <div class="row">
+                <blockquote>{{ import.description|trans }}</blockquote>
+                <p>{{ 'import.readability.how_to'|trans }}</p>
+
+                <div class="col s12">
+                    {{ form_start(form, {'method': 'POST'}) }}
+                        {{ form_errors(form) }}
+                        <div class="row">
+                            <div class="file-field input-field col s12">
+                                {{ form_errors(form.file) }}
+                                <div class="btn">
+                                    <span>{{ form.file.vars.label|trans }}</span>
+                                    {{ form_widget(form.file) }}
+                                </div>
+                                <div class="file-path-wrapper">
+                                    <input class="file-path validate" type="text">
+                                </div>
+                            </div>
+                            <div class="input-field col s6 with-checkbox">
+                                <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
+                                {{ form_widget(form.mark_as_read) }}
+                                {{ form_label(form.mark_as_read) }}
+                            </div>
+                        </div>
+
+                        {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
+
+                        {{ form_rest(form) }}
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}
diff --git a/tests/Wallabag/ImportBundle/fixtures/readability.json b/tests/Wallabag/ImportBundle/fixtures/readability.json
new file mode 100644
index 00000000..34379905
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/fixtures/readability.json
@@ -0,0 +1,25 @@
+{
+    "bookmarks": [
+        {
+            "article__excerpt": "When Twitter started it had so much promise to change the way we communicate. But now it has been ruined by the amount of garbage and hate we have to wade through. It&#x2019;s like that polluted&hellip;",
+            "favorite": false,
+            "date_archived": null,
+            "article__url": "https://venngage.com/blog/hashtags-are-worthless/",
+            "date_added": "2016-08-25T12:05:00",
+            "date_favorited": null,
+            "article__title": "We Looked At 167,943 Tweets & Found Out Hashtags Are Worthless",
+            "archive": false
+        },
+        {
+            "article__excerpt": "TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data. Here&#x2019;s a demo and the code for the infinite&hellip;",
+            "favorite": false,
+            "date_archived": "2016-08-26T12:21:54",
+            "article__url": "https://developers.google.com/web/updates/2016/07/infinite-scroller?imm_mid=0e6839&cmp=em-webops-na-na-newsltr_20160805",
+            "date_added": "2016-08-06T05:35:26",
+            "date_favorited": null,
+            "article__title": "Complexities of an infinite scroller | Web Updates",
+            "archive": true
+        }
+   ],
+    "recommendations": []
+}