aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorJeremy Benoist <jeremy.benoist@gmail.com>2016-09-04 21:49:21 +0200
committerJeremy Benoist <jeremy.benoist@gmail.com>2016-09-11 21:58:31 +0200
commitc98db1b653b5dc8b701422190b02d9fbf10c4e68 (patch)
tree5f02de371264143a4e81231ad16f605435cb4d22
parentef75e1220ebb76a8df019d946460ad612759f0bb (diff)
downloadwallabag-c98db1b653b5dc8b701422190b02d9fbf10c4e68.tar.gz
wallabag-c98db1b653b5dc8b701422190b02d9fbf10c4e68.tar.zst
wallabag-c98db1b653b5dc8b701422190b02d9fbf10c4e68.zip
Convert other imports to Rabbit
-rw-r--r--app/config/config.yml43
-rw-r--r--src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php (renamed from src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php)14
-rw-r--r--src/Wallabag/ImportBundle/Controller/PocketController.php2
-rw-r--r--src/Wallabag/ImportBundle/Controller/ReadabilityController.php6
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagController.php2
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php8
-rw-r--r--src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php8
-rw-r--r--src/Wallabag/ImportBundle/Import/AbstractImport.php83
-rw-r--r--src/Wallabag/ImportBundle/Import/PocketImport.php107
-rw-r--r--src/Wallabag/ImportBundle/Import/ReadabilityImport.php127
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagImport.php118
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV1Import.php20
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV2Import.php20
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/services.yml23
14 files changed, 326 insertions, 255 deletions
diff --git a/app/config/config.yml b/app/config/config.yml
index fa829637..d39bef95 100644
--- a/app/config/config.yml
+++ b/app/config/config.yml
@@ -226,13 +226,28 @@ old_sound_rabbit_mq:
226 vhost: / 226 vhost: /
227 lazy: true 227 lazy: true
228 producers: 228 producers:
229 wallabag_pocket: 229 import_pocket:
230 connection: default 230 connection: default
231 exchange_options: 231 exchange_options:
232 name: 'wallabag.import.pocket' 232 name: 'wallabag.import.pocket'
233 type: topic 233 type: topic
234 import_readability:
235 connection: default
236 exchange_options:
237 name: 'wallabag.import.readability'
238 type: topic
239 import_wallabag_v1:
240 connection: default
241 exchange_options:
242 name: 'wallabag.import.wallabag_v1'
243 type: topic
244 import_wallabag_v2:
245 connection: default
246 exchange_options:
247 name: 'wallabag.import.wallabag_v2'
248 type: topic
234 consumers: 249 consumers:
235 wallabag_pocket: 250 import_pocket:
236 connection: default 251 connection: default
237 exchange_options: 252 exchange_options:
238 name: 'wallabag.import.pocket' 253 name: 'wallabag.import.pocket'
@@ -240,3 +255,27 @@ old_sound_rabbit_mq:
240 queue_options: 255 queue_options:
241 name: 'wallabag.import.pocket' 256 name: 'wallabag.import.pocket'
242 callback: wallabag_import.consumer.pocket 257 callback: wallabag_import.consumer.pocket
258 import_readability:
259 connection: default
260 exchange_options:
261 name: 'wallabag.import.readability'
262 type: topic
263 queue_options:
264 name: 'wallabag.import.readability'
265 callback: wallabag_import.consumer.readability
266 import_wallabag_v1:
267 connection: default
268 exchange_options:
269 name: 'wallabag.import.wallabag_v1'
270 type: topic
271 queue_options:
272 name: 'wallabag.import.wallabag_v1'
273 callback: wallabag_import.consumer.wallabag_v1
274 import_wallabag_v2:
275 connection: default
276 exchange_options:
277 name: 'wallabag.import.wallabag_v2'
278 type: topic
279 queue_options:
280 name: 'wallabag.import.wallabag_v2'
281 callback: wallabag_import.consumer.wallabag_v2
diff --git a/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php
index 239e7446..8a8cf45d 100644
--- a/src/Wallabag/ImportBundle/Consumer/AMPQ/PocketConsumer.php
+++ b/src/Wallabag/ImportBundle/Consumer/AMPQ/EntryConsumer.php
@@ -5,23 +5,23 @@ namespace Wallabag\ImportBundle\Consumer\AMPQ;
5use Doctrine\ORM\EntityManager; 5use Doctrine\ORM\EntityManager;
6use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface; 6use OldSound\RabbitMqBundle\RabbitMq\ConsumerInterface;
7use PhpAmqpLib\Message\AMQPMessage; 7use PhpAmqpLib\Message\AMQPMessage;
8use Wallabag\ImportBundle\Import\PocketImport; 8use Wallabag\ImportBundle\Import\AbstractImport;
9use Wallabag\UserBundle\Repository\UserRepository; 9use Wallabag\UserBundle\Repository\UserRepository;
10use Psr\Log\LoggerInterface; 10use Psr\Log\LoggerInterface;
11use Psr\Log\NullLogger; 11use Psr\Log\NullLogger;
12 12
13class PocketConsumer implements ConsumerInterface 13class EntryConsumer implements ConsumerInterface
14{ 14{
15 private $em; 15 private $em;
16 private $userRepository; 16 private $userRepository;
17 private $pocketImport; 17 private $import;
18 private $logger; 18 private $logger;
19 19
20 public function __construct(EntityManager $em, UserRepository $userRepository, PocketImport $pocketImport, LoggerInterface $logger = null) 20 public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null)
21 { 21 {
22 $this->em = $em; 22 $this->em = $em;
23 $this->userRepository = $userRepository; 23 $this->userRepository = $userRepository;
24 $this->pocketImport = $pocketImport; 24 $this->import = $import;
25 $this->logger = $logger ?: new NullLogger(); 25 $this->logger = $logger ?: new NullLogger();
26 } 26 }
27 27
@@ -41,9 +41,9 @@ class PocketConsumer implements ConsumerInterface
41 return; 41 return;
42 } 42 }
43 43
44 $this->pocketImport->setUser($user); 44 $this->import->setUser($user);
45 45
46 $entry = $this->pocketImport->parseEntry($storedEntry); 46 $entry = $this->import->parseEntry($storedEntry);
47 47
48 if (null === $entry) { 48 if (null === $entry) {
49 $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]); 49 $this->logger->warning('Unable to parse entry', ['entry' => $storedEntry]);
diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php
index a2dcd8a7..57c007c3 100644
--- a/src/Wallabag/ImportBundle/Controller/PocketController.php
+++ b/src/Wallabag/ImportBundle/Controller/PocketController.php
@@ -21,7 +21,7 @@ class PocketController extends Controller
21 $pocket->setUser($this->getUser()); 21 $pocket->setUser($this->getUser());
22 22
23 if ($this->get('craue_config')->get('rabbitmq')) { 23 if ($this->get('craue_config')->get('rabbitmq')) {
24 $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.wallabag_pocket_producer')); 24 $pocket->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_pocket_producer'));
25 } 25 }
26 26
27 return $pocket; 27 return $pocket;
diff --git a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
index b61aa99c..ee875a40 100644
--- a/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
+++ b/src/Wallabag/ImportBundle/Controller/ReadabilityController.php
@@ -18,6 +18,11 @@ class ReadabilityController extends Controller
18 $form->handleRequest($request); 18 $form->handleRequest($request);
19 19
20 $readability = $this->get('wallabag_import.readability.import'); 20 $readability = $this->get('wallabag_import.readability.import');
21 $readability->setUser($this->getUser());
22
23 if ($this->get('craue_config')->get('rabbitmq')) {
24 $readability->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_readability_producer'));
25 }
21 26
22 if ($form->isValid()) { 27 if ($form->isValid()) {
23 $file = $form->get('file')->getData(); 28 $file = $form->get('file')->getData();
@@ -26,7 +31,6 @@ class ReadabilityController extends Controller
26 31
27 if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { 32 if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
28 $res = $readability 33 $res = $readability
29 ->setUser($this->getUser())
30 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) 34 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
31 ->setMarkAsRead($markAsRead) 35 ->setMarkAsRead($markAsRead)
32 ->import(); 36 ->import();
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagController.php b/src/Wallabag/ImportBundle/Controller/WallabagController.php
index 76ced0d2..1e6114c5 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagController.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagController.php
@@ -38,6 +38,7 @@ abstract class WallabagController extends Controller
38 $form->handleRequest($request); 38 $form->handleRequest($request);
39 39
40 $wallabag = $this->getImportService(); 40 $wallabag = $this->getImportService();
41 $wallabag->setUser($this->getUser());
41 42
42 if ($form->isValid()) { 43 if ($form->isValid()) {
43 $file = $form->get('file')->getData(); 44 $file = $form->get('file')->getData();
@@ -46,7 +47,6 @@ abstract class WallabagController extends Controller
46 47
47 if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { 48 if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) {
48 $res = $wallabag 49 $res = $wallabag
49 ->setUser($this->getUser())
50 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) 50 ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name)
51 ->setMarkAsRead($markAsRead) 51 ->setMarkAsRead($markAsRead)
52 ->import(); 52 ->import();
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
index 3e748d57..f80aec3a 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagV1Controller.php
@@ -12,7 +12,13 @@ class WallabagV1Controller extends WallabagController
12 */ 12 */
13 protected function getImportService() 13 protected function getImportService()
14 { 14 {
15 return $this->get('wallabag_import.wallabag_v1.import'); 15 $service = $this->get('wallabag_import.wallabag_v1.import');
16
17 if ($this->get('craue_config')->get('rabbitmq')) {
18 $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer'));
19 }
20
21 return $service;
16 } 22 }
17 23
18 /** 24 /**
diff --git a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
index c2a42165..063cddd9 100644
--- a/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
+++ b/src/Wallabag/ImportBundle/Controller/WallabagV2Controller.php
@@ -12,7 +12,13 @@ class WallabagV2Controller extends WallabagController
12 */ 12 */
13 protected function getImportService() 13 protected function getImportService()
14 { 14 {
15 return $this->get('wallabag_import.wallabag_v2.import'); 15 $service = $this->get('wallabag_import.wallabag_v2.import');
16
17 if ($this->get('craue_config')->get('rabbitmq')) {
18 $service->setRabbitmqProducer($this->get('old_sound_rabbit_mq.import_wallabag_v2_producer'));
19 }
20
21 return $service;
16 } 22 }
17 23
18 /** 24 /**
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index 14377a35..5b9d65d7 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -7,12 +7,17 @@ use Psr\Log\NullLogger;
7use Doctrine\ORM\EntityManager; 7use Doctrine\ORM\EntityManager;
8use Wallabag\CoreBundle\Helper\ContentProxy; 8use Wallabag\CoreBundle\Helper\ContentProxy;
9use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
10use Symfony\Component\Security\Core\User\UserInterface;
11use OldSound\RabbitMqBundle\RabbitMq\Producer;
10 12
11abstract class AbstractImport implements ImportInterface 13abstract class AbstractImport implements ImportInterface
12{ 14{
13 protected $em; 15 protected $em;
14 protected $logger; 16 protected $logger;
15 protected $contentProxy; 17 protected $contentProxy;
18 protected $producer;
19 protected $user;
20 protected $markAsRead;
16 21
17 public function __construct(EntityManager $em, ContentProxy $contentProxy) 22 public function __construct(EntityManager $em, ContentProxy $contentProxy)
18 { 23 {
@@ -27,6 +32,48 @@ abstract class AbstractImport implements ImportInterface
27 } 32 }
28 33
29 /** 34 /**
35 * Set RabbitMQ Producer to send each entry to a queue.
36 * This method should be called when user has enabled RabbitMQ.
37 *
38 * @param Producer $producer
39 */
40 public function setRabbitmqProducer(Producer $producer)
41 {
42 $this->producer = $producer;
43 }
44
45 /**
46 * Set current user.
47 * Could the current *connected* user or one retrieve by the consumer.
48 *
49 * @param UserInterface $user
50 */
51 public function setUser(UserInterface $user)
52 {
53 $this->user = $user;
54 }
55
56 /**
57 * Set whether articles must be all marked as read.
58 *
59 * @param bool $markAsRead
60 */
61 public function setMarkAsRead($markAsRead)
62 {
63 $this->markAsRead = $markAsRead;
64
65 return $this;
66 }
67
68 /**
69 * Get whether articles must be all marked as read.
70 */
71 public function getMarkAsRead()
72 {
73 return $this->markAsRead;
74 }
75
76 /**
30 * Fetch content from the ContentProxy (using graby). 77 * Fetch content from the ContentProxy (using graby).
31 * If it fails return false instead of the updated entry. 78 * If it fails return false instead of the updated entry.
32 * 79 *
@@ -44,4 +91,40 @@ abstract class AbstractImport implements ImportInterface
44 return false; 91 return false;
45 } 92 }
46 } 93 }
94
95 /**
96 * Parse and insert all given entries.
97 *
98 * @param $entries
99 */
100 protected function parseEntries($entries)
101 {
102 $i = 1;
103
104 foreach ($entries as $importedEntry) {
105 $entry = $this->parseEntry($importedEntry);
106
107 if (null === $entry) {
108 continue;
109 }
110
111 // flush every 20 entries
112 if (($i % 20) === 0) {
113 $this->em->flush();
114 $this->em->clear($entry);
115 }
116 ++$i;
117 }
118
119 $this->em->flush();
120 }
121
122 /**
123 * Parse one entry.
124 *
125 * @param array $importedEntry
126 *
127 * @return Entry
128 */
129 abstract public function parseEntry(array $importedEntry);
47} 130}
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 27df4917..dd0ddd72 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -2,25 +2,20 @@
2 2
3namespace Wallabag\ImportBundle\Import; 3namespace Wallabag\ImportBundle\Import;
4 4
5use OldSound\RabbitMqBundle\RabbitMq\Producer;
6use Psr\Log\NullLogger; 5use Psr\Log\NullLogger;
7use Doctrine\ORM\EntityManager; 6use Doctrine\ORM\EntityManager;
8use GuzzleHttp\Client; 7use GuzzleHttp\Client;
9use GuzzleHttp\Exception\RequestException; 8use GuzzleHttp\Exception\RequestException;
10use Symfony\Component\Security\Core\User\UserInterface;
11use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
12use Wallabag\CoreBundle\Helper\ContentProxy; 10use Wallabag\CoreBundle\Helper\ContentProxy;
13use Craue\ConfigBundle\Util\Config; 11use Craue\ConfigBundle\Util\Config;
14 12
15class PocketImport extends AbstractImport 13class PocketImport extends AbstractImport
16{ 14{
17 private $user;
18 private $client; 15 private $client;
19 private $consumerKey; 16 private $consumerKey;
20 private $skippedEntries = 0; 17 private $skippedEntries = 0;
21 private $importedEntries = 0; 18 private $importedEntries = 0;
22 private $markAsRead;
23 private $producer;
24 protected $accessToken; 19 protected $accessToken;
25 20
26 public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) 21 public function __construct(EntityManager $em, ContentProxy $contentProxy, Config $craueConfig)
@@ -32,28 +27,6 @@ class PocketImport extends AbstractImport
32 } 27 }
33 28
34 /** 29 /**
35 * Set RabbitMQ Producer to send each entry to a queue.
36 * This method should be called when user has enabled RabbitMQ.
37 *
38 * @param Producer $producer
39 */
40 public function setRabbitmqProducer(Producer $producer)
41 {
42 $this->producer = $producer;
43 }
44
45 /**
46 * Set current user.
47 * Could the current *connected* user or one retrieve by the consumer.
48 *
49 * @param UserInterface $user
50 */
51 public function setUser(UserInterface $user)
52 {
53 $this->user = $user;
54 }
55
56 /**
57 * {@inheritdoc} 30 * {@inheritdoc}
58 */ 31 */
59 public function getName() 32 public function getName()
@@ -139,26 +112,6 @@ class PocketImport extends AbstractImport
139 } 112 }
140 113
141 /** 114 /**
142 * Set whether articles must be all marked as read.
143 *
144 * @param bool $markAsRead
145 */
146 public function setMarkAsRead($markAsRead)
147 {
148 $this->markAsRead = $markAsRead;
149
150 return $this;
151 }
152
153 /**
154 * Get whether articles must be all marked as read.
155 */
156 public function getMarkAsRead()
157 {
158 return $this->markAsRead;
159 }
160
161 /**
162 * {@inheritdoc} 115 * {@inheritdoc}
163 */ 116 */
164 public function import() 117 public function import()
@@ -217,37 +170,9 @@ class PocketImport extends AbstractImport
217 $this->client = $client; 170 $this->client = $client;
218 } 171 }
219 172
220 /** 173 public function parseEntry(array $importedEntry)
221 * @see https://getpocket.com/developer/docs/v3/retrieve
222 *
223 * @param array $entries
224 */
225 private function parseEntries(array $entries)
226 {
227 $i = 1;
228
229 foreach ($entries as $pocketEntry) {
230 $entry = $this->parseEntry($pocketEntry);
231
232 if (null === $entry) {
233 continue;
234 }
235
236 // flush every 20 entries
237 if (($i % 20) === 0) {
238 $this->em->flush();
239 $this->em->clear($entry);
240 }
241
242 ++$i;
243 }
244
245 $this->em->flush();
246 }
247
248 public function parseEntry(array $pocketEntry)
249 { 174 {
250 $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; 175 $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
251 176
252 $existingEntry = $this->em 177 $existingEntry = $this->em
253 ->getRepository('WallabagCoreBundle:Entry') 178 ->getRepository('WallabagCoreBundle:Entry')
@@ -270,34 +195,34 @@ class PocketImport extends AbstractImport
270 } 195 }
271 196
272 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted 197 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
273 if ($pocketEntry['status'] == 1 || $this->markAsRead) { 198 if ($importedEntry['status'] == 1 || $this->markAsRead) {
274 $entry->setArchived(true); 199 $entry->setArchived(true);
275 } 200 }
276 201
277 // 0 or 1 - 1 If the item is starred 202 // 0 or 1 - 1 If the item is starred
278 if ($pocketEntry['favorite'] == 1) { 203 if ($importedEntry['favorite'] == 1) {
279 $entry->setStarred(true); 204 $entry->setStarred(true);
280 } 205 }
281 206
282 $title = 'Untitled'; 207 $title = 'Untitled';
283 if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { 208 if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
284 $title = $pocketEntry['resolved_title']; 209 $title = $importedEntry['resolved_title'];
285 } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { 210 } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
286 $title = $pocketEntry['given_title']; 211 $title = $importedEntry['given_title'];
287 } 212 }
288 213
289 $entry->setTitle($title); 214 $entry->setTitle($title);
290 $entry->setUrl($url); 215 $entry->setUrl($url);
291 216
292 // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image 217 // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
293 if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { 218 if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
294 $entry->setPreviewPicture($pocketEntry['images'][1]['src']); 219 $entry->setPreviewPicture($importedEntry['images'][1]['src']);
295 } 220 }
296 221
297 if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { 222 if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
298 $this->contentProxy->assignTagsToEntry( 223 $this->contentProxy->assignTagsToEntry(
299 $entry, 224 $entry,
300 array_keys($pocketEntry['tags']) 225 array_keys($importedEntry['tags'])
301 ); 226 );
302 } 227 }
303 228
@@ -315,17 +240,17 @@ class PocketImport extends AbstractImport
315 */ 240 */
316 public function parseEntriesForProducer($entries) 241 public function parseEntriesForProducer($entries)
317 { 242 {
318 foreach ($entries as $pocketEntry) { 243 foreach ($entries as $importedEntry) {
319 // set userId for the producer (it won't know which user is connected) 244 // set userId for the producer (it won't know which user is connected)
320 $pocketEntry['userId'] = $this->user->getId(); 245 $importedEntry['userId'] = $this->user->getId();
321 246
322 if ($this->markAsRead) { 247 if ($this->markAsRead) {
323 $pocketEntry['status'] = 1; 248 $importedEntry['status'] = 1;
324 } 249 }
325 250
326 ++$this->importedEntries; 251 ++$this->importedEntries;
327 252
328 $this->producer->publish(json_encode($pocketEntry)); 253 $this->producer->publish(json_encode($importedEntry));
329 } 254 }
330 } 255 }
331} 256}
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
index c7cfe15d..18a6631a 100644
--- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -7,24 +7,9 @@ use Wallabag\UserBundle\Entity\User;
7 7
8class ReadabilityImport extends AbstractImport 8class ReadabilityImport extends AbstractImport
9{ 9{
10 private $user;
11 private $skippedEntries = 0; 10 private $skippedEntries = 0;
12 private $importedEntries = 0; 11 private $importedEntries = 0;
13 private $filepath; 12 private $filepath;
14 private $markAsRead;
15
16 /**
17 * We define the user in a custom call because on the import command there is no logged in user.
18 * So we can't retrieve user from the `security.token_storage` service.
19 *
20 * @param User $user
21 */
22 public function setUser(User $user)
23 {
24 $this->user = $user;
25
26 return $this;
27 }
28 13
29 /** 14 /**
30 * {@inheritdoc} 15 * {@inheritdoc}
@@ -63,26 +48,6 @@ class ReadabilityImport extends AbstractImport
63 } 48 }
64 49
65 /** 50 /**
66 * Set whether articles must be all marked as read.
67 *
68 * @param bool $markAsRead
69 */
70 public function setMarkAsRead($markAsRead)
71 {
72 $this->markAsRead = $markAsRead;
73
74 return $this;
75 }
76
77 /**
78 * Get whether articles must be all marked as read.
79 */
80 public function getMarkAsRead()
81 {
82 return $this->markAsRead;
83 }
84
85 /**
86 * {@inheritdoc} 51 * {@inheritdoc}
87 */ 52 */
88 public function getSummary() 53 public function getSummary()
@@ -116,54 +81,76 @@ class ReadabilityImport extends AbstractImport
116 return false; 81 return false;
117 } 82 }
118 83
84 if ($this->producer) {
85 $this->parseEntriesForProducer($data['bookmarks']);
86
87 return true;
88 }
89
119 $this->parseEntries($data['bookmarks']); 90 $this->parseEntries($data['bookmarks']);
120 91
121 return true; 92 return true;
122 } 93 }
123 94
95 public function parseEntry(array $importedEntry)
96 {
97 $existingEntry = $this->em
98 ->getRepository('WallabagCoreBundle:Entry')
99 ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
100
101 if (false !== $existingEntry) {
102 ++$this->skippedEntries;
103
104 return;
105 }
106
107 $data = [
108 'title' => $importedEntry['article__title'],
109 'url' => $importedEntry['article__url'],
110 'content_type' => '',
111 'language' => '',
112 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
113 'is_starred' => $importedEntry['favorite'],
114 ];
115
116 $entry = $this->fetchContent(
117 new Entry($this->user),
118 $data['url'],
119 $data
120 );
121
122 // jump to next entry in case of problem while getting content
123 if (false === $entry) {
124 ++$this->skippedEntries;
125
126 return;
127 }
128
129 $entry->setArchived($data['is_archived']);
130 $entry->setStarred($data['is_starred']);
131
132 $this->em->persist($entry);
133 ++$this->importedEntries;
134
135 return $entry;
136 }
137
124 /** 138 /**
125 * Parse and insert all given entries. 139 * Faster parse entries for Producer.
140 * We don't care to make check at this time. They'll be done by the consumer.
126 * 141 *
127 * @param $entries 142 * @param array $entries
128 */ 143 */
129 protected function parseEntries($entries) 144 protected function parseEntriesForProducer($entries)
130 { 145 {
131 $i = 1;
132
133 foreach ($entries as $importedEntry) { 146 foreach ($entries as $importedEntry) {
134 $existingEntry = $this->em 147 // set userId for the producer (it won't know which user is connected)
135 ->getRepository('WallabagCoreBundle:Entry') 148 $importedEntry['userId'] = $this->user->getId();
136 ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
137
138 if (false !== $existingEntry) {
139 ++$this->skippedEntries;
140 continue;
141 }
142 149
143 $data = [ 150 if ($this->markAsRead) {
144 'title' => $importedEntry['article__title'], 151 $importedEntry['archive'] = 1;
145 'url' => $importedEntry['article__url'],
146 'content_type' => '',
147 'language' => '',
148 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
149 'is_starred' => $importedEntry['favorite'],
150 ];
151
152 $entry = $this->fetchContent(
153 new Entry($this->user),
154 $data['url'],
155 $data
156 );
157
158 // jump to next entry in case of problem while getting content
159 if (false === $entry) {
160 ++$this->skippedEntries;
161 continue;
162 } 152 }
163 $entry->setArchived($data['is_archived']);
164 $entry->setStarred($data['is_starred']);
165 153
166 $this->em->persist($entry);
167 ++$this->importedEntries; 154 ++$this->importedEntries;
168 155
169 // flush every 20 entries 156 // flush every 20 entries
diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php
index 581ec178..6ad14e8c 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagImport.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php
@@ -11,7 +11,6 @@ abstract class WallabagImport extends AbstractImport
11 protected $skippedEntries = 0; 11 protected $skippedEntries = 0;
12 protected $importedEntries = 0; 12 protected $importedEntries = 0;
13 protected $filepath; 13 protected $filepath;
14 protected $markAsRead;
15 // untitled in all languages from v1 14 // untitled in all languages from v1
16 protected $untitled = [ 15 protected $untitled = [
17 'Untitled', 16 'Untitled',
@@ -29,19 +28,6 @@ abstract class WallabagImport extends AbstractImport
29 ]; 28 ];
30 29
31 /** 30 /**
32 * We define the user in a custom call because on the import command there is no logged in user.
33 * So we can't retrieve user from the `security.token_storage` service.
34 *
35 * @param User $user
36 */
37 public function setUser(User $user)
38 {
39 $this->user = $user;
40
41 return $this;
42 }
43
44 /**
45 * {@inheritdoc} 31 * {@inheritdoc}
46 */ 32 */
47 abstract public function getName(); 33 abstract public function getName();
@@ -79,6 +65,12 @@ abstract class WallabagImport extends AbstractImport
79 return false; 65 return false;
80 } 66 }
81 67
68 if ($this->producer) {
69 $this->parseEntriesForProducer($data);
70
71 return true;
72 }
73
82 $this->parseEntries($data); 74 $this->parseEntries($data);
83 75
84 return true; 76 return true;
@@ -108,85 +100,61 @@ abstract class WallabagImport extends AbstractImport
108 } 100 }
109 101
110 /** 102 /**
111 * Set whether articles must be all marked as read. 103 * {@inheritdoc}
112 *
113 * @param bool $markAsRead
114 */ 104 */
115 public function setMarkAsRead($markAsRead) 105 public function parseEntry(array $importedEntry)
116 { 106 {
117 $this->markAsRead = $markAsRead; 107 $existingEntry = $this->em
108 ->getRepository('WallabagCoreBundle:Entry')
109 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
118 110
119 return $this; 111 if (false !== $existingEntry) {
120 } 112 ++$this->skippedEntries;
121 113
122 /** 114 return;
123 * Parse and insert all given entries. 115 }
124 * 116
125 * @param $entries 117 $data = $this->prepareEntry($importedEntry);
126 */
127 protected function parseEntries($entries)
128 {
129 $i = 1;
130 118
131 foreach ($entries as $importedEntry) { 119 $entry = $this->fetchContent(
132 $existingEntry = $this->em 120 new Entry($this->user),
133 ->getRepository('WallabagCoreBundle:Entry') 121 $importedEntry['url'],
134 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); 122 $data
123 );
135 124
136 if (false !== $existingEntry) { 125 // jump to next entry in case of problem while getting content
137 ++$this->skippedEntries; 126 if (false === $entry) {
138 continue; 127 ++$this->skippedEntries;
139 }
140 128
141 $data = $this->prepareEntry($importedEntry, $this->markAsRead); 129 return;
130 }
142 131
143 $entry = $this->fetchContent( 132 if (array_key_exists('tags', $data)) {
144 new Entry($this->user), 133 $this->contentProxy->assignTagsToEntry(
145 $importedEntry['url'], 134 $entry,
146 $data 135 $data['tags']
147 ); 136 );
137 }
148 138
149 // jump to next entry in case of problem while getting content 139 if (isset($importedEntry['preview_picture'])) {
150 if (false === $entry) { 140 $entry->setPreviewPicture($importedEntry['preview_picture']);
151 ++$this->skippedEntries;
152 continue;
153 }
154
155 if (array_key_exists('tags', $data)) {
156 $this->contentProxy->assignTagsToEntry(
157 $entry,
158 $data['tags']
159 );
160 }
161
162 if (isset($importedEntry['preview_picture'])) {
163 $entry->setPreviewPicture($importedEntry['preview_picture']);
164 }
165
166 $entry->setArchived($data['is_archived']);
167 $entry->setStarred($data['is_starred']);
168
169 $this->em->persist($entry);
170 ++$this->importedEntries;
171
172 // flush every 20 entries
173 if (($i % 20) === 0) {
174 $this->em->flush();
175 }
176 ++$i;
177 } 141 }
178 142
179 $this->em->flush(); 143 $entry->setArchived($data['is_archived']);
180 $this->em->clear(); 144 $entry->setStarred($data['is_starred']);
145
146 $this->em->persist($entry);
147 ++$this->importedEntries;
148
149 return $entry;
181 } 150 }
182 151
183 /** 152 /**
184 * This should return a cleaned array for a given entry to be given to `updateEntry`. 153 * This should return a cleaned array for a given entry to be given to `updateEntry`.
185 * 154 *
186 * @param array $entry Data from the imported file 155 * @param array $entry Data from the imported file
187 * @param bool $markAsRead Should we mark as read content?
188 * 156 *
189 * @return array 157 * @return array
190 */ 158 */
191 abstract protected function prepareEntry($entry = [], $markAsRead = false); 159 abstract protected function prepareEntry($entry = []);
192} 160}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
index 6cf3467a..86734652 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
@@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport
31 /** 31 /**
32 * {@inheritdoc} 32 * {@inheritdoc}
33 */ 33 */
34 protected function prepareEntry($entry = [], $markAsRead = false) 34 protected function prepareEntry($entry = [])
35 { 35 {
36 $data = [ 36 $data = [
37 'title' => $entry['title'], 37 'title' => $entry['title'],
@@ -39,7 +39,7 @@ class WallabagV1Import extends WallabagImport
39 'url' => $entry['url'], 39 'url' => $entry['url'],
40 'content_type' => '', 40 'content_type' => '',
41 'language' => '', 41 'language' => '',
42 'is_archived' => $entry['is_read'] || $markAsRead, 42 'is_archived' => $entry['is_read'] || $this->markAsRead,
43 'is_starred' => $entry['is_fav'], 43 'is_starred' => $entry['is_fav'],
44 'tags' => '', 44 'tags' => '',
45 ]; 45 ];
@@ -56,4 +56,20 @@ class WallabagV1Import extends WallabagImport
56 56
57 return $data; 57 return $data;
58 } 58 }
59
60 protected function parseEntriesForProducer($entries)
61 {
62 foreach ($entries as $importedEntry) {
63 // set userId for the producer (it won't know which user is connected)
64 $importedEntry['userId'] = $this->user->getId();
65
66 if ($this->markAsRead) {
67 $importedEntry['is_read'] = 1;
68 }
69
70 ++$this->importedEntries;
71
72 $this->producer->publish(json_encode($importedEntry));
73 }
74 }
59} 75}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
index d0035b63..faf4236f 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
@@ -31,12 +31,28 @@ class WallabagV2Import extends WallabagImport
31 /** 31 /**
32 * {@inheritdoc} 32 * {@inheritdoc}
33 */ 33 */
34 protected function prepareEntry($entry = [], $markAsRead = false) 34 protected function prepareEntry($entry = [])
35 { 35 {
36 return [ 36 return [
37 'html' => $entry['content'], 37 'html' => $entry['content'],
38 'content_type' => $entry['mimetype'], 38 'content_type' => $entry['mimetype'],
39 'is_archived' => ($entry['is_archived'] || $markAsRead), 39 'is_archived' => ($entry['is_archived'] || $this->markAsRead),
40 ] + $entry; 40 ] + $entry;
41 } 41 }
42
43 protected function parseEntriesForProducer($entries)
44 {
45 foreach ($entries as $importedEntry) {
46 // set userId for the producer (it won't know which user is connected)
47 $importedEntry['userId'] = $this->user->getId();
48
49 if ($this->markAsRead) {
50 $importedEntry['is_archived'] = 1;
51 }
52
53 ++$this->importedEntries;
54
55 $this->producer->publish(json_encode($importedEntry));
56 }
57 }
42} 58}
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index fe388b26..cad44e71 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -1,11 +1,32 @@
1services: 1services:
2 wallabag_import.consumer.pocket: 2 wallabag_import.consumer.pocket:
3 class: Wallabag\ImportBundle\Consumer\AMPQ\PocketConsumer 3 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
4 arguments: 4 arguments:
5 - "@doctrine.orm.entity_manager" 5 - "@doctrine.orm.entity_manager"
6 - "@wallabag_user.user_repository" 6 - "@wallabag_user.user_repository"
7 - "@wallabag_import.pocket.import" 7 - "@wallabag_import.pocket.import"
8 - "@logger" 8 - "@logger"
9 wallabag_import.consumer.readability:
10 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
11 arguments:
12 - "@doctrine.orm.entity_manager"
13 - "@wallabag_user.user_repository"
14 - "@wallabag_import.readability.import"
15 - "@logger"
16 wallabag_import.consumer.wallabag_v1:
17 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
18 arguments:
19 - "@doctrine.orm.entity_manager"
20 - "@wallabag_user.user_repository"
21 - "@wallabag_import.wallabag_v1.import"
22 - "@logger"
23 wallabag_import.consumer.wallabag_v2:
24 class: Wallabag\ImportBundle\Consumer\AMPQ\EntryConsumer
25 arguments:
26 - "@doctrine.orm.entity_manager"
27 - "@wallabag_user.user_repository"
28 - "@wallabag_import.wallabag_v2.import"
29 - "@logger"
9 30
10 wallabag_import.chain: 31 wallabag_import.chain:
11 class: Wallabag\ImportBundle\Import\ImportChain 32 class: Wallabag\ImportBundle\Import\ImportChain