From 03e3753f6bd36f12c0757c76b49b683c49de48ae Mon Sep 17 00:00:00 2001
From: Jeremy Benoist <jeremy.benoist@gmail.com>
Date: Thu, 1 Sep 2016 08:00:30 +0200
Subject: Add Readability import

Based on the JSON export instead of the API (which will be shutting down by the September 30, 2016)
---
 .../Controller/ReadabilityController.php           |  65 ++++++++
 .../ImportBundle/Import/ReadabilityImport.php      | 181 +++++++++++++++++++++
 .../ImportBundle/Resources/config/services.yml     |  10 ++
 .../Resources/views/Readability/index.html.twig    |  43 +++++
 4 files changed, 299 insertions(+)
 create mode 100644 src/Wallabag/ImportBundle/Controller/ReadabilityController.php
 create mode 100644 src/Wallabag/ImportBundle/Import/ReadabilityImport.php
 create mode 100644 src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig

(limited to 'src/Wallabag/ImportBundle')

diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
new file mode 100644
index 00000000..b61aa99c
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
@@ -0,0 +1,65 @@
+<?php
+
+namespace Wallabag\ImportBundle\Controller;
+
+use Symfony\Bundle\FrameworkBundle\Controller\Controller;
+use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
+use Symfony\Component\HttpFoundation\Request;
+use Wallabag\ImportBundle\Form\Type\UploadImportType;
+
+class ReadabilityController extends Controller
+{
+    /**
+     * @Route("/readability", name="import_readability")
+     */
+    public function indexAction(Request $request)
+    {
+        $form = $this->createForm(UploadImportType::class);
+        $form->handleRequest($request);
+
+        $readability = $this->get('wallabag_import.readability.import');
+
+        if ($form->isValid()) {
+            $file = $form->get('file')->getData();
+            $markAsRead = $form->get('mark_as_read')->getData();
+            $name = 'readability_'.$this->getUser()->getId().'.json';
+
+            if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
+                $res = $readability
+                    ->setUser($this->getUser())
+                    ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
+                    ->setMarkAsRead($markAsRead)
+                    ->import();
+
+                $message = 'flashes.import.notice.failed';
+
+                if (true === $res) {
+                    $summary = $readability->getSummary();
+                    $message = $this->get('translator')->trans('flashes.import.notice.summary', [
+                        '%imported%' => $summary['imported'],
+                        '%skipped%' => $summary['skipped'],
+                    ]);
+
+                    unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
+                }
+
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    $message
+                );
+
+                return $this->redirect($this->generateUrl('homepage'));
+            } else {
+                $this->get('session')->getFlashBag()->add(
+                    'notice',
+                    'flashes.import.notice.failed_on_file'
+                );
+            }
+        }
+
+        return $this->render('WallabagImportBundle:Readability:index.html.twig', [
+            'form' => $form->createView(),
+            'import' => $readability,
+        ]);
+    }
+}
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
new file mode 100644
index 00000000..abea81a7
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -0,0 +1,181 @@
+<?php
+
+namespace Wallabag\ImportBundle\Import;
+
+use Wallabag\CoreBundle\Entity\Entry;
+use Wallabag\UserBundle\Entity\User;
+
+class ReadabilityImport extends AbstractImport
+{
+    private $user;
+    private $client;
+    private $skippedEntries = 0;
+    private $importedEntries = 0;
+    private $filepath;
+    private $markAsRead;
+
+    /**
+     * We define the user in a custom call because on the import command there is no logged in user.
+     * So we can't retrieve user from the `security.token_storage` service.
+     *
+     * @param User $user
+     */
+    public function setUser(User $user)
+    {
+        $this->user = $user;
+
+        return $this;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getName()
+    {
+        return 'Readability';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getUrl()
+    {
+        return 'import_readability';
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getDescription()
+    {
+        return 'import.readability.description';
+    }
+
+    /**
+     * Set file path to the json file.
+     *
+     * @param string $filepath
+     */
+    public function setFilepath($filepath)
+    {
+        $this->filepath = $filepath;
+
+        return $this;
+    }
+
+    /**
+     * Set whether articles must be all marked as read.
+     *
+     * @param bool $markAsRead
+     */
+    public function setMarkAsRead($markAsRead)
+    {
+        $this->markAsRead = $markAsRead;
+
+        return $this;
+    }
+
+    /**
+     * Get whether articles must be all marked as read.
+     */
+    public function getMarkAsRead()
+    {
+        return $this->markAsRead;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function getSummary()
+    {
+        return [
+            'skipped' => $this->skippedEntries,
+            'imported' => $this->importedEntries,
+        ];
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function import()
+    {
+        if (!$this->user) {
+            $this->logger->error('ReadabilityImport: user is not defined');
+
+            return false;
+        }
+
+        if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
+            $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]);
+
+            return false;
+        }
+
+        $data = json_decode(file_get_contents($this->filepath), true);
+
+        if (empty($data) || empty($data['bookmarks'])) {
+            return false;
+        }
+
+        $this->parseEntries($data['bookmarks']);
+
+        return true;
+    }
+
+    /**
+     * Parse and insert all given entries.
+     *
+     * @param $entries
+     */
+    protected function parseEntries($entries)
+    {
+        $i = 1;
+
+        foreach ($entries as $importedEntry) {
+            $existingEntry = $this->em
+                ->getRepository('WallabagCoreBundle:Entry')
+                ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
+
+            if (false !== $existingEntry) {
+                ++$this->skippedEntries;
+                continue;
+            }
+
+            $data = [
+                'title' => $importedEntry['article__title'],
+                // 'html' => $importedEntry['article__excerpt'],
+                'url' => $importedEntry['article__url'],
+                'content_type' => '',
+                'language' => '',
+                'is_archived' => $importedEntry['archive'] || $this->markAsRead,
+                'is_starred' => $importedEntry['favorite'],
+            ];
+
+            $entry = $this->fetchContent(
+                new Entry($this->user),
+                $data['url'],
+                $data
+            );
+
+            // jump to next entry in case of problem while getting content
+            if (false === $entry) {
+                ++$this->skippedEntries;
+                continue;
+            }
+            $entry->setArchived($data['is_archived']);
+            $entry->setStarred($data['is_starred']);
+
+            $this->em->persist($entry);
+            ++$this->importedEntries;
+
+            // flush every 20 entries
+            if (($i % 20) === 0) {
+                $this->em->flush();
+                $this->em->clear($entry);
+            }
+            ++$i;
+        }
+
+        $this->em->flush();
+    }
+}
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index 86b44cb3..520d43af 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -43,3 +43,13 @@ services:
             - [ setLogger, [ "@logger" ]]
         tags:
             -  { name: wallabag_import.import, alias: wallabag_v2 }
+
+    wallabag_import.readability.import:
+        class: Wallabag\ImportBundle\Import\ReadabilityImport
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_core.content_proxy"
+        calls:
+            - [ setLogger, [ "@logger" ]]
+        tags:
+            -  { name: wallabag_import.import, alias: readability }
diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
new file mode 100644
index 00000000..f527d309
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
@@ -0,0 +1,43 @@
+{% extends "WallabagCoreBundle::layout.html.twig" %}
+
+{% block title %}{{ 'import.readability.page_title'|trans }}{% endblock %}
+
+{% block content %}
+<div class="row">
+    <div class="col s12">
+        <div class="card-panel settings">
+            <div class="row">
+                <blockquote>{{ import.description|trans }}</blockquote>
+                <p>{{ 'import.readability.how_to'|trans }}</p>
+
+                <div class="col s12">
+                    {{ form_start(form, {'method': 'POST'}) }}
+                        {{ form_errors(form) }}
+                        <div class="row">
+                            <div class="file-field input-field col s12">
+                                {{ form_errors(form.file) }}
+                                <div class="btn">
+                                    <span>{{ form.file.vars.label|trans }}</span>
+                                    {{ form_widget(form.file) }}
+                                </div>
+                                <div class="file-path-wrapper">
+                                    <input class="file-path validate" type="text">
+                                </div>
+                            </div>
+                            <div class="input-field col s6 with-checkbox">
+                                <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
+                                {{ form_widget(form.mark_as_read) }}
+                                {{ form_label(form.mark_as_read) }}
+                            </div>
+                        </div>
+
+                        {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
+
+                        {{ form_rest(form) }}
+                    </form>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}
-- 
cgit v1.2.3


From a1a107705948c724aca7326f8550c336a25a6364 Mon Sep 17 00:00:00 2001
From: Jeremy Benoist <jeremy.benoist@gmail.com>
Date: Thu, 1 Sep 2016 09:57:02 +0200
Subject: Add tests on ReadabilityImport

---
 src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 2 --
 1 file changed, 2 deletions(-)

(limited to 'src/Wallabag/ImportBundle')

diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
index abea81a7..37b160c5 100644
--- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -8,7 +8,6 @@ use Wallabag\UserBundle\Entity\User;
 class ReadabilityImport extends AbstractImport
 {
     private $user;
-    private $client;
     private $skippedEntries = 0;
     private $importedEntries = 0;
     private $filepath;
@@ -143,7 +142,6 @@ class ReadabilityImport extends AbstractImport
 
             $data = [
                 'title' => $importedEntry['article__title'],
-                // 'html' => $importedEntry['article__excerpt'],
                 'url' => $importedEntry['article__url'],
                 'content_type' => '',
                 'language' => '',
-- 
cgit v1.2.3