aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorJeremy Benoist <jeremy.benoist@gmail.com>2016-09-01 08:00:30 +0200
committerJeremy Benoist <jeremy.benoist@gmail.com>2016-09-01 08:00:30 +0200
commit03e3753f6bd36f12c0757c76b49b683c49de48ae (patch)
tree1f5f5b7b35e0f610371e1ca83bbbad34571325ba
parentcdd3010b478c9ca818dd6d22d03c81ef4a5ab208 (diff)
downloadwallabag-03e3753f6bd36f12c0757c76b49b683c49de48ae.tar.gz
wallabag-03e3753f6bd36f12c0757c76b49b683c49de48ae.tar.zst
wallabag-03e3753f6bd36f12c0757c76b49b683c49de48ae.zip
Add Readability import
Based on the JSON export instead of the API (which will be shutting down by the September 30, 2016)
-rw-r--r--src/Wallabag/CoreBundle/Resources/translations/messages.en.yml4
-rw-r--r--src/Wallabag/ImportBundle/Controller/ReadabilityController.php65
-rw-r--r--src/Wallabag/ImportBundle/Import/ReadabilityImport.php181
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/services.yml10
-rw-r--r--src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig43
-rw-r--r--tests/Wallabag/ImportBundle/fixtures/readability.json25
6 files changed, 328 insertions, 0 deletions
diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
index 220c4d9c..f2bf9aed 100644
--- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
+++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml
@@ -341,6 +341,10 @@ import:
341 wallabag_v2: 341 wallabag_v2:
342 page_title: 'Import > Wallabag v2' 342 page_title: 'Import > Wallabag v2'
343 description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.' 343 description: 'This importer will import all your wallabag v2 articles. Go to All articles, then, on the export sidebar, click on "JSON". You will have a "All articles.json" file.'
344 readability:
345 page_title: 'Import > Readability'
346 description: 'This importer will import all your Readability articles. On the tools (https://www.readability.com/tools/) page, click on "Export your data" in the "Data Export" section. You will received an email to download a json (which does not end with .json in fact)'
347 how_to: 'Please select your Readability export and click on the below button to upload and import it.'
344 348
345developer: 349developer:
346 page_title: 'Developer' 350 page_title: 'Developer'
diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
new file mode 100644
index 00000000..b61aa99c
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
@@ -0,0 +1,65 @@
1<?php
2
3namespace Wallabag\ImportBundle\Controller;
4
5use Symfony\Bundle\FrameworkBundle\Controller\Controller;
6use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
7use Symfony\Component\HttpFoundation\Request;
8use Wallabag\ImportBundle\Form\Type\UploadImportType;
9
10class ReadabilityController extends Controller
11{
12 /**
13 * @Route("/readability", name="import_readability")
14 */
15 public function indexAction(Request $request)
16 {
17 $form = $this->createForm(UploadImportType::class);
18 $form->handleRequest($request);
19
20 $readability = $this->get('wallabag_import.readability.import');
21
22 if ($form->isValid()) {
23 $file = $form->get('file')->getData();
24 $markAsRead = $form->get('mark_as_read')->getData();
25 $name = 'readability_'.$this->getUser()->getId().'.json';
26
27 if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
28 $res = $readability
29 ->setUser($this->getUser())
30 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
31 ->setMarkAsRead($markAsRead)
32 ->import();
33
34 $message = 'flashes.import.notice.failed';
35
36 if (true === $res) {
37 $summary = $readability->getSummary();
38 $message = $this->get('translator')->trans('flashes.import.notice.summary', [
39 '%imported%' => $summary['imported'],
40 '%skipped%' => $summary['skipped'],
41 ]);
42
43 unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name);
44 }
45
46 $this->get('session')->getFlashBag()->add(
47 'notice',
48 $message
49 );
50
51 return $this->redirect($this->generateUrl('homepage'));
52 } else {
53 $this->get('session')->getFlashBag()->add(
54 'notice',
55 'flashes.import.notice.failed_on_file'
56 );
57 }
58 }
59
60 return $this->render('WallabagImportBundle:Readability:index.html.twig', [
61 'form' => $form->createView(),
62 'import' => $readability,
63 ]);
64 }
65}
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
new file mode 100644
index 00000000..abea81a7
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -0,0 +1,181 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5use Wallabag\CoreBundle\Entity\Entry;
6use Wallabag\UserBundle\Entity\User;
7
8class ReadabilityImport extends AbstractImport
9{
10 private $user;
11 private $client;
12 private $skippedEntries = 0;
13 private $importedEntries = 0;
14 private $filepath;
15 private $markAsRead;
16
17 /**
18 * We define the user in a custom call because on the import command there is no logged in user.
19 * So we can't retrieve user from the `security.token_storage` service.
20 *
21 * @param User $user
22 */
23 public function setUser(User $user)
24 {
25 $this->user = $user;
26
27 return $this;
28 }
29
30 /**
31 * {@inheritdoc}
32 */
33 public function getName()
34 {
35 return 'Readability';
36 }
37
38 /**
39 * {@inheritdoc}
40 */
41 public function getUrl()
42 {
43 return 'import_readability';
44 }
45
46 /**
47 * {@inheritdoc}
48 */
49 public function getDescription()
50 {
51 return 'import.readability.description';
52 }
53
54 /**
55 * Set file path to the json file.
56 *
57 * @param string $filepath
58 */
59 public function setFilepath($filepath)
60 {
61 $this->filepath = $filepath;
62
63 return $this;
64 }
65
66 /**
67 * Set whether articles must be all marked as read.
68 *
69 * @param bool $markAsRead
70 */
71 public function setMarkAsRead($markAsRead)
72 {
73 $this->markAsRead = $markAsRead;
74
75 return $this;
76 }
77
78 /**
79 * Get whether articles must be all marked as read.
80 */
81 public function getMarkAsRead()
82 {
83 return $this->markAsRead;
84 }
85
86 /**
87 * {@inheritdoc}
88 */
89 public function getSummary()
90 {
91 return [
92 'skipped' => $this->skippedEntries,
93 'imported' => $this->importedEntries,
94 ];
95 }
96
97 /**
98 * {@inheritdoc}
99 */
100 public function import()
101 {
102 if (!$this->user) {
103 $this->logger->error('ReadabilityImport: user is not defined');
104
105 return false;
106 }
107
108 if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
109 $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]);
110
111 return false;
112 }
113
114 $data = json_decode(file_get_contents($this->filepath), true);
115
116 if (empty($data) || empty($data['bookmarks'])) {
117 return false;
118 }
119
120 $this->parseEntries($data['bookmarks']);
121
122 return true;
123 }
124
125 /**
126 * Parse and insert all given entries.
127 *
128 * @param $entries
129 */
130 protected function parseEntries($entries)
131 {
132 $i = 1;
133
134 foreach ($entries as $importedEntry) {
135 $existingEntry = $this->em
136 ->getRepository('WallabagCoreBundle:Entry')
137 ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
138
139 if (false !== $existingEntry) {
140 ++$this->skippedEntries;
141 continue;
142 }
143
144 $data = [
145 'title' => $importedEntry['article__title'],
146 // 'html' => $importedEntry['article__excerpt'],
147 'url' => $importedEntry['article__url'],
148 'content_type' => '',
149 'language' => '',
150 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
151 'is_starred' => $importedEntry['favorite'],
152 ];
153
154 $entry = $this->fetchContent(
155 new Entry($this->user),
156 $data['url'],
157 $data
158 );
159
160 // jump to next entry in case of problem while getting content
161 if (false === $entry) {
162 ++$this->skippedEntries;
163 continue;
164 }
165 $entry->setArchived($data['is_archived']);
166 $entry->setStarred($data['is_starred']);
167
168 $this->em->persist($entry);
169 ++$this->importedEntries;
170
171 // flush every 20 entries
172 if (($i % 20) === 0) {
173 $this->em->flush();
174 $this->em->clear($entry);
175 }
176 ++$i;
177 }
178
179 $this->em->flush();
180 }
181}
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index 86b44cb3..520d43af 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -43,3 +43,13 @@ services:
43 - [ setLogger, [ "@logger" ]] 43 - [ setLogger, [ "@logger" ]]
44 tags: 44 tags:
45 - { name: wallabag_import.import, alias: wallabag_v2 } 45 - { name: wallabag_import.import, alias: wallabag_v2 }
46
47 wallabag_import.readability.import:
48 class: Wallabag\ImportBundle\Import\ReadabilityImport
49 arguments:
50 - "@doctrine.orm.entity_manager"
51 - "@wallabag_core.content_proxy"
52 calls:
53 - [ setLogger, [ "@logger" ]]
54 tags:
55 - { name: wallabag_import.import, alias: readability }
diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
new file mode 100644
index 00000000..f527d309
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
@@ -0,0 +1,43 @@
1{% extends "WallabagCoreBundle::layout.html.twig" %}
2
3{% block title %}{{ 'import.readability.page_title'|trans }}{% endblock %}
4
5{% block content %}
6<div class="row">
7 <div class="col s12">
8 <div class="card-panel settings">
9 <div class="row">
10 <blockquote>{{ import.description|trans }}</blockquote>
11 <p>{{ 'import.readability.how_to'|trans }}</p>
12
13 <div class="col s12">
14 {{ form_start(form, {'method': 'POST'}) }}
15 {{ form_errors(form) }}
16 <div class="row">
17 <div class="file-field input-field col s12">
18 {{ form_errors(form.file) }}
19 <div class="btn">
20 <span>{{ form.file.vars.label|trans }}</span>
21 {{ form_widget(form.file) }}
22 </div>
23 <div class="file-path-wrapper">
24 <input class="file-path validate" type="text">
25 </div>
26 </div>
27 <div class="input-field col s6 with-checkbox">
28 <h6>{{ 'import.form.mark_as_read_title'|trans }}</h6>
29 {{ form_widget(form.mark_as_read) }}
30 {{ form_label(form.mark_as_read) }}
31 </div>
32 </div>
33
34 {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }}
35
36 {{ form_rest(form) }}
37 </form>
38 </div>
39 </div>
40 </div>
41 </div>
42</div>
43{% endblock %}
diff --git a/tests/Wallabag/ImportBundle/fixtures/readability.json b/tests/Wallabag/ImportBundle/fixtures/readability.json
new file mode 100644
index 00000000..34379905
--- /dev/null
+++ b/tests/Wallabag/ImportBundle/fixtures/readability.json
@@ -0,0 +1,25 @@
1{
2 "bookmarks": [
3 {
4 "article__excerpt": "When Twitter started it had so much promise to change the way we communicate. But now it has been ruined by the amount of garbage and hate we have to wade through. It&#x2019;s like that polluted&hellip;",
5 "favorite": false,
6 "date_archived": null,
7 "article__url": "https://venngage.com/blog/hashtags-are-worthless/",
8 "date_added": "2016-08-25T12:05:00",
9 "date_favorited": null,
10 "article__title": "We Looked At 167,943 Tweets & Found Out Hashtags Are Worthless",
11 "archive": false
12 },
13 {
14 "article__excerpt": "TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data. Here&#x2019;s a demo and the code for the infinite&hellip;",
15 "favorite": false,
16 "date_archived": "2016-08-26T12:21:54",
17 "article__url": "https://developers.google.com/web/updates/2016/07/infinite-scroller?imm_mid=0e6839&cmp=em-webops-na-na-newsltr_20160805",
18 "date_added": "2016-08-06T05:35:26",
19 "date_favorited": null,
20 "article__title": "Complexities of an infinite scroller | Web Updates",
21 "archive": true
22 }
23 ],
24 "recommendations": []
25}