aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/Wallabag/ImportBundle/Import
diff options
context:
space:
mode:
Diffstat (limited to 'src/Wallabag/ImportBundle/Import')
-rw-r--r--src/Wallabag/ImportBundle/Import/AbstractImport.php144
-rw-r--r--src/Wallabag/ImportBundle/Import/PocketImport.php218
-rw-r--r--src/Wallabag/ImportBundle/Import/ReadabilityImport.php149
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagImport.php131
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV1Import.php15
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV2Import.php14
6 files changed, 361 insertions, 310 deletions
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index 14377a35..a1a14576 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -7,12 +7,21 @@ use Psr\Log\NullLogger;
7use Doctrine\ORM\EntityManager; 7use Doctrine\ORM\EntityManager;
8use Wallabag\CoreBundle\Helper\ContentProxy; 8use Wallabag\CoreBundle\Helper\ContentProxy;
9use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
10use Wallabag\CoreBundle\Entity\Tag;
11use Wallabag\UserBundle\Entity\User;
12use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
10 13
11abstract class AbstractImport implements ImportInterface 14abstract class AbstractImport implements ImportInterface
12{ 15{
13 protected $em; 16 protected $em;
14 protected $logger; 17 protected $logger;
15 protected $contentProxy; 18 protected $contentProxy;
19 protected $producer;
20 protected $user;
21 protected $markAsRead;
22 protected $skippedEntries = 0;
23 protected $importedEntries = 0;
24 protected $queuedEntries = 0;
16 25
17 public function __construct(EntityManager $em, ContentProxy $contentProxy) 26 public function __construct(EntityManager $em, ContentProxy $contentProxy)
18 { 27 {
@@ -27,21 +36,150 @@ abstract class AbstractImport implements ImportInterface
27 } 36 }
28 37
29 /** 38 /**
39 * Set RabbitMQ/Redis Producer to send each entry to a queue.
40 * This method should be called when user has enabled RabbitMQ.
41 *
42 * @param ProducerInterface $producer
43 */
44 public function setProducer(ProducerInterface $producer)
45 {
46 $this->producer = $producer;
47 }
48
49 /**
50 * Set current user.
51 * Could the current *connected* user or one retrieve by the consumer.
52 *
53 * @param User $user
54 */
55 public function setUser(User $user)
56 {
57 $this->user = $user;
58 }
59
60 /**
61 * Set whether articles must be all marked as read.
62 *
63 * @param bool $markAsRead
64 */
65 public function setMarkAsRead($markAsRead)
66 {
67 $this->markAsRead = $markAsRead;
68
69 return $this;
70 }
71
72 /**
73 * Get whether articles must be all marked as read.
74 */
75 public function getMarkAsRead()
76 {
77 return $this->markAsRead;
78 }
79
80 /**
30 * Fetch content from the ContentProxy (using graby). 81 * Fetch content from the ContentProxy (using graby).
31 * If it fails return false instead of the updated entry. 82 * If it fails return the given entry to be saved in all case (to avoid user to loose the content).
32 * 83 *
33 * @param Entry $entry Entry to update 84 * @param Entry $entry Entry to update
34 * @param string $url Url to grab content for 85 * @param string $url Url to grab content for
35 * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url 86 * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
36 * 87 *
37 * @return Entry|false 88 * @return Entry
38 */ 89 */
39 protected function fetchContent(Entry $entry, $url, array $content = []) 90 protected function fetchContent(Entry $entry, $url, array $content = [])
40 { 91 {
41 try { 92 try {
42 return $this->contentProxy->updateEntry($entry, $url, $content); 93 return $this->contentProxy->updateEntry($entry, $url, $content);
43 } catch (\Exception $e) { 94 } catch (\Exception $e) {
44 return false; 95 return $entry;
96 }
97 }
98
99 /**
100 * Parse and insert all given entries.
101 *
102 * @param $entries
103 */
104 protected function parseEntries($entries)
105 {
106 $i = 1;
107
108 foreach ($entries as $importedEntry) {
109 $entry = $this->parseEntry($importedEntry);
110
111 if (null === $entry) {
112 continue;
113 }
114
115 // flush every 20 entries
116 if (($i % 20) === 0) {
117 $this->em->flush();
118
119 // clear only affected entities
120 $this->em->clear(Entry::class);
121 $this->em->clear(Tag::class);
122 }
123 ++$i;
124 }
125
126 $this->em->flush();
127 }
128
129 /**
130 * Parse entries and send them to the queue.
131 * It should just be a simple loop on all item, no call to the database should be done
132 * to speedup queuing.
133 *
134 * Faster parse entries for Producer.
135 * We don't care to make check at this time. They'll be done by the consumer.
136 *
137 * @param array $entries
138 */
139 protected function parseEntriesForProducer(array $entries)
140 {
141 foreach ($entries as $importedEntry) {
142 // set userId for the producer (it won't know which user is connected)
143 $importedEntry['userId'] = $this->user->getId();
144
145 if ($this->markAsRead) {
146 $importedEntry = $this->setEntryAsRead($importedEntry);
147 }
148
149 ++$this->queuedEntries;
150
151 $this->producer->publish(json_encode($importedEntry));
45 } 152 }
46 } 153 }
154
155 /**
156 * {@inheritdoc}
157 */
158 public function getSummary()
159 {
160 return [
161 'skipped' => $this->skippedEntries,
162 'imported' => $this->importedEntries,
163 'queued' => $this->queuedEntries,
164 ];
165 }
166
167 /**
168 * Parse one entry.
169 *
170 * @param array $importedEntry
171 *
172 * @return Entry
173 */
174 abstract public function parseEntry(array $importedEntry);
175
176 /**
177 * Set current imported entry to archived / read.
178 * Implementation is different accross all imports.
179 *
180 * @param array $importedEntry
181 *
182 * @return array
183 */
184 abstract protected function setEntryAsRead(array $importedEntry);
47} 185}
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index a6f905b1..e00eb44b 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -6,31 +6,34 @@ use Psr\Log\NullLogger;
6use Doctrine\ORM\EntityManager; 6use Doctrine\ORM\EntityManager;
7use GuzzleHttp\Client; 7use GuzzleHttp\Client;
8use GuzzleHttp\Exception\RequestException; 8use GuzzleHttp\Exception\RequestException;
9use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface;
10use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
11use Wallabag\CoreBundle\Helper\ContentProxy; 10use Wallabag\CoreBundle\Helper\ContentProxy;
12use Craue\ConfigBundle\Util\Config;
13 11
14class PocketImport extends AbstractImport 12class PocketImport extends AbstractImport
15{ 13{
16 private $user;
17 private $client; 14 private $client;
18 private $consumerKey; 15 private $accessToken;
19 private $skippedEntries = 0;
20 private $importedEntries = 0;
21 private $markAsRead;
22 protected $accessToken;
23 16
24 public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) 17 const NB_ELEMENTS = 5000;
18
19 public function __construct(EntityManager $em, ContentProxy $contentProxy)
25 { 20 {
26 $this->user = $tokenStorage->getToken()->getUser();
27 $this->em = $em; 21 $this->em = $em;
28 $this->contentProxy = $contentProxy; 22 $this->contentProxy = $contentProxy;
29 $this->consumerKey = $craueConfig->get('pocket_consumer_key');
30 $this->logger = new NullLogger(); 23 $this->logger = new NullLogger();
31 } 24 }
32 25
33 /** 26 /**
27 * Only used for test purpose.
28 *
29 * @return string
30 */
31 public function getAccessToken()
32 {
33 return $this->accessToken;
34 }
35
36 /**
34 * {@inheritdoc} 37 * {@inheritdoc}
35 */ 38 */
36 public function getName() 39 public function getName()
@@ -66,7 +69,7 @@ class PocketImport extends AbstractImport
66 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', 69 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request',
67 [ 70 [
68 'body' => json_encode([ 71 'body' => json_encode([
69 'consumer_key' => $this->consumerKey, 72 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
70 'redirect_uri' => $redirectUri, 73 'redirect_uri' => $redirectUri,
71 ]), 74 ]),
72 ] 75 ]
@@ -96,7 +99,7 @@ class PocketImport extends AbstractImport
96 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', 99 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize',
97 [ 100 [
98 'body' => json_encode([ 101 'body' => json_encode([
99 'consumer_key' => $this->consumerKey, 102 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
100 'code' => $code, 103 'code' => $code,
101 ]), 104 ]),
102 ] 105 ]
@@ -116,38 +119,22 @@ class PocketImport extends AbstractImport
116 } 119 }
117 120
118 /** 121 /**
119 * Set whether articles must be all marked as read.
120 *
121 * @param bool $markAsRead
122 */
123 public function setMarkAsRead($markAsRead)
124 {
125 $this->markAsRead = $markAsRead;
126
127 return $this;
128 }
129
130 /**
131 * Get whether articles must be all marked as read.
132 */
133 public function getMarkAsRead()
134 {
135 return $this->markAsRead;
136 }
137
138 /**
139 * {@inheritdoc} 122 * {@inheritdoc}
140 */ 123 */
141 public function import() 124 public function import($offset = 0)
142 { 125 {
126 static $run = 0;
127
143 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', 128 $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get',
144 [ 129 [
145 'body' => json_encode([ 130 'body' => json_encode([
146 'consumer_key' => $this->consumerKey, 131 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(),
147 'access_token' => $this->accessToken, 132 'access_token' => $this->accessToken,
148 'detailType' => 'complete', 133 'detailType' => 'complete',
149 'state' => 'all', 134 'state' => 'all',
150 'sort' => 'oldest', 135 'sort' => 'newest',
136 'count' => self::NB_ELEMENTS,
137 'offset' => $offset,
151 ]), 138 ]),
152 ] 139 ]
153 ); 140 );
@@ -162,20 +149,24 @@ class PocketImport extends AbstractImport
162 149
163 $entries = $response->json(); 150 $entries = $response->json();
164 151
165 $this->parseEntries($entries['list']); 152 if ($this->producer) {
153 $this->parseEntriesForProducer($entries['list']);
154 } else {
155 $this->parseEntries($entries['list']);
156 }
166 157
167 return true; 158 // if we retrieve exactly the amount of items requested it means we can get more
168 } 159 // re-call import and offset item by the amount previous received:
160 // - first call get 5k offset 0
161 // - second call get 5k offset 5k
162 // - and so on
163 if (count($entries['list']) === self::NB_ELEMENTS) {
164 ++$run;
169 165
170 /** 166 return $this->import(self::NB_ELEMENTS * $run);
171 * {@inheritdoc} 167 }
172 */ 168
173 public function getSummary() 169 return true;
174 {
175 return [
176 'skipped' => $this->skippedEntries,
177 'imported' => $this->importedEntries,
178 ];
179 } 170 }
180 171
181 /** 172 /**
@@ -189,77 +180,74 @@ class PocketImport extends AbstractImport
189 } 180 }
190 181
191 /** 182 /**
192 * @see https://getpocket.com/developer/docs/v3/retrieve 183 * {@inheritdoc}
193 * 184 *
194 * @param $entries 185 * @see https://getpocket.com/developer/docs/v3/retrieve
195 */ 186 */
196 private function parseEntries($entries) 187 public function parseEntry(array $importedEntry)
197 { 188 {
198 $i = 1; 189 $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url'];
199 190
200 foreach ($entries as $pocketEntry) { 191 $existingEntry = $this->em
201 $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; 192 ->getRepository('WallabagCoreBundle:Entry')
202 193 ->findByUrlAndUserId($url, $this->user->getId());
203 $existingEntry = $this->em 194
204 ->getRepository('WallabagCoreBundle:Entry') 195 if (false !== $existingEntry) {
205 ->findByUrlAndUserId($url, $this->user->getId()); 196 ++$this->skippedEntries;
206 197
207 if (false !== $existingEntry) { 198 return;
208 ++$this->skippedEntries; 199 }
209 continue; 200
210 } 201 $entry = new Entry($this->user);
211 202 $entry->setUrl($url);
212 $entry = new Entry($this->user); 203
213 $entry = $this->fetchContent($entry, $url); 204 // update entry with content (in case fetching failed, the given entry will be return)
214 205 $entry = $this->fetchContent($entry, $url);
215 // jump to next entry in case of problem while getting content 206
216 if (false === $entry) { 207 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
217 ++$this->skippedEntries; 208 $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
218 continue; 209
219 } 210 // 0 or 1 - 1 If the item is starred
220 211 $entry->setStarred($importedEntry['favorite'] == 1);
221 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted 212
222 if ($pocketEntry['status'] == 1 || $this->markAsRead) { 213 $title = 'Untitled';
223 $entry->setArchived(true); 214 if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') {
224 } 215 $title = $importedEntry['resolved_title'];
225 216 } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') {
226 // 0 or 1 - 1 If the item is starred 217 $title = $importedEntry['given_title'];
227 if ($pocketEntry['favorite'] == 1) { 218 }
228 $entry->setStarred(true); 219
229 } 220 $entry->setTitle($title);
230 221
231 $title = 'Untitled'; 222 // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image
232 if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { 223 if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) {
233 $title = $pocketEntry['resolved_title']; 224 $entry->setPreviewPicture($importedEntry['images'][1]['src']);
234 } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { 225 }
235 $title = $pocketEntry['given_title']; 226
236 } 227 if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) {
237 228 $this->contentProxy->assignTagsToEntry(
238 $entry->setTitle($title); 229 $entry,
239 230 array_keys($importedEntry['tags'])
240 // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image 231 );
241 if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) {
242 $entry->setPreviewPicture($pocketEntry['images'][1]['src']);
243 }
244
245 if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) {
246 $this->contentProxy->assignTagsToEntry(
247 $entry,
248 array_keys($pocketEntry['tags'])
249 );
250 }
251
252 $this->em->persist($entry);
253 ++$this->importedEntries;
254
255 // flush every 20 entries
256 if (($i % 20) === 0) {
257 $this->em->flush();
258 }
259 ++$i;
260 } 232 }
261 233
262 $this->em->flush(); 234 if (!empty($importedEntry['time_added'])) {
263 $this->em->clear(); 235 $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added']));
236 }
237
238 $this->em->persist($entry);
239 ++$this->importedEntries;
240
241 return $entry;
242 }
243
244 /**
245 * {@inheritdoc}
246 */
247 protected function setEntryAsRead(array $importedEntry)
248 {
249 $importedEntry['status'] = '1';
250
251 return $importedEntry;
264 } 252 }
265} 253}
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
index c7cfe15d..fa2b7053 100644
--- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -3,28 +3,10 @@
3namespace Wallabag\ImportBundle\Import; 3namespace Wallabag\ImportBundle\Import;
4 4
5use Wallabag\CoreBundle\Entity\Entry; 5use Wallabag\CoreBundle\Entity\Entry;
6use Wallabag\UserBundle\Entity\User;
7 6
8class ReadabilityImport extends AbstractImport 7class ReadabilityImport extends AbstractImport
9{ 8{
10 private $user;
11 private $skippedEntries = 0;
12 private $importedEntries = 0;
13 private $filepath; 9 private $filepath;
14 private $markAsRead;
15
16 /**
17 * We define the user in a custom call because on the import command there is no logged in user.
18 * So we can't retrieve user from the `security.token_storage` service.
19 *
20 * @param User $user
21 */
22 public function setUser(User $user)
23 {
24 $this->user = $user;
25
26 return $this;
27 }
28 10
29 /** 11 /**
30 * {@inheritdoc} 12 * {@inheritdoc}
@@ -63,37 +45,6 @@ class ReadabilityImport extends AbstractImport
63 } 45 }
64 46
65 /** 47 /**
66 * Set whether articles must be all marked as read.
67 *
68 * @param bool $markAsRead
69 */
70 public function setMarkAsRead($markAsRead)
71 {
72 $this->markAsRead = $markAsRead;
73
74 return $this;
75 }
76
77 /**
78 * Get whether articles must be all marked as read.
79 */
80 public function getMarkAsRead()
81 {
82 return $this->markAsRead;
83 }
84
85 /**
86 * {@inheritdoc}
87 */
88 public function getSummary()
89 {
90 return [
91 'skipped' => $this->skippedEntries,
92 'imported' => $this->importedEntries,
93 ];
94 }
95
96 /**
97 * {@inheritdoc} 48 * {@inheritdoc}
98 */ 49 */
99 public function import() 50 public function import()
@@ -116,64 +67,66 @@ class ReadabilityImport extends AbstractImport
116 return false; 67 return false;
117 } 68 }
118 69
70 if ($this->producer) {
71 $this->parseEntriesForProducer($data['bookmarks']);
72
73 return true;
74 }
75
119 $this->parseEntries($data['bookmarks']); 76 $this->parseEntries($data['bookmarks']);
120 77
121 return true; 78 return true;
122 } 79 }
123 80
124 /** 81 /**
125 * Parse and insert all given entries. 82 * {@inheritdoc}
126 *
127 * @param $entries
128 */ 83 */
129 protected function parseEntries($entries) 84 public function parseEntry(array $importedEntry)
130 { 85 {
131 $i = 1; 86 $existingEntry = $this->em
132 87 ->getRepository('WallabagCoreBundle:Entry')
133 foreach ($entries as $importedEntry) { 88 ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId());
134 $existingEntry = $this->em 89
135 ->getRepository('WallabagCoreBundle:Entry') 90 if (false !== $existingEntry) {
136 ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); 91 ++$this->skippedEntries;
137 92
138 if (false !== $existingEntry) { 93 return;
139 ++$this->skippedEntries;
140 continue;
141 }
142
143 $data = [
144 'title' => $importedEntry['article__title'],
145 'url' => $importedEntry['article__url'],
146 'content_type' => '',
147 'language' => '',
148 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
149 'is_starred' => $importedEntry['favorite'],
150 ];
151
152 $entry = $this->fetchContent(
153 new Entry($this->user),
154 $data['url'],
155 $data
156 );
157
158 // jump to next entry in case of problem while getting content
159 if (false === $entry) {
160 ++$this->skippedEntries;
161 continue;
162 }
163 $entry->setArchived($data['is_archived']);
164 $entry->setStarred($data['is_starred']);
165
166 $this->em->persist($entry);
167 ++$this->importedEntries;
168
169 // flush every 20 entries
170 if (($i % 20) === 0) {
171 $this->em->flush();
172 }
173 ++$i;
174 } 94 }
175 95
176 $this->em->flush(); 96 $data = [
177 $this->em->clear(); 97 'title' => $importedEntry['article__title'],
98 'url' => $importedEntry['article__url'],
99 'content_type' => '',
100 'language' => '',
101 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
102 'is_starred' => $importedEntry['favorite'],
103 'created_at' => $importedEntry['date_added'],
104 ];
105
106 $entry = new Entry($this->user);
107 $entry->setUrl($data['url']);
108 $entry->setTitle($data['title']);
109
110 // update entry with content (in case fetching failed, the given entry will be return)
111 $entry = $this->fetchContent($entry, $data['url'], $data);
112
113 $entry->setArchived($data['is_archived']);
114 $entry->setStarred($data['is_starred']);
115 $entry->setCreatedAt(new \DateTime($data['created_at']));
116
117 $this->em->persist($entry);
118 ++$this->importedEntries;
119
120 return $entry;
121 }
122
123 /**
124 * {@inheritdoc}
125 */
126 protected function setEntryAsRead(array $importedEntry)
127 {
128 $importedEntry['archive'] = 1;
129
130 return $importedEntry;
178 } 131 }
179} 132}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php
index 581ec178..043bb0a2 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagImport.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php
@@ -3,15 +3,10 @@
3namespace Wallabag\ImportBundle\Import; 3namespace Wallabag\ImportBundle\Import;
4 4
5use Wallabag\CoreBundle\Entity\Entry; 5use Wallabag\CoreBundle\Entity\Entry;
6use Wallabag\UserBundle\Entity\User;
7 6
8abstract class WallabagImport extends AbstractImport 7abstract class WallabagImport extends AbstractImport
9{ 8{
10 protected $user;
11 protected $skippedEntries = 0;
12 protected $importedEntries = 0;
13 protected $filepath; 9 protected $filepath;
14 protected $markAsRead;
15 // untitled in all languages from v1 10 // untitled in all languages from v1
16 protected $untitled = [ 11 protected $untitled = [
17 'Untitled', 12 'Untitled',
@@ -29,19 +24,6 @@ abstract class WallabagImport extends AbstractImport
29 ]; 24 ];
30 25
31 /** 26 /**
32 * We define the user in a custom call because on the import command there is no logged in user.
33 * So we can't retrieve user from the `security.token_storage` service.
34 *
35 * @param User $user
36 */
37 public function setUser(User $user)
38 {
39 $this->user = $user;
40
41 return $this;
42 }
43
44 /**
45 * {@inheritdoc} 27 * {@inheritdoc}
46 */ 28 */
47 abstract public function getName(); 29 abstract public function getName();
@@ -79,23 +61,18 @@ abstract class WallabagImport extends AbstractImport
79 return false; 61 return false;
80 } 62 }
81 63
64 if ($this->producer) {
65 $this->parseEntriesForProducer($data);
66
67 return true;
68 }
69
82 $this->parseEntries($data); 70 $this->parseEntries($data);
83 71
84 return true; 72 return true;
85 } 73 }
86 74
87 /** 75 /**
88 * {@inheritdoc}
89 */
90 public function getSummary()
91 {
92 return [
93 'skipped' => $this->skippedEntries,
94 'imported' => $this->importedEntries,
95 ];
96 }
97
98 /**
99 * Set file path to the json file. 76 * Set file path to the json file.
100 * 77 *
101 * @param string $filepath 78 * @param string $filepath
@@ -108,85 +85,59 @@ abstract class WallabagImport extends AbstractImport
108 } 85 }
109 86
110 /** 87 /**
111 * Set whether articles must be all marked as read. 88 * {@inheritdoc}
112 *
113 * @param bool $markAsRead
114 */ 89 */
115 public function setMarkAsRead($markAsRead) 90 public function parseEntry(array $importedEntry)
116 { 91 {
117 $this->markAsRead = $markAsRead; 92 $existingEntry = $this->em
93 ->getRepository('WallabagCoreBundle:Entry')
94 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
118 95
119 return $this; 96 if (false !== $existingEntry) {
120 } 97 ++$this->skippedEntries;
121 98
122 /** 99 return;
123 * Parse and insert all given entries. 100 }
124 *
125 * @param $entries
126 */
127 protected function parseEntries($entries)
128 {
129 $i = 1;
130 101
131 foreach ($entries as $importedEntry) { 102 $data = $this->prepareEntry($importedEntry);
132 $existingEntry = $this->em
133 ->getRepository('WallabagCoreBundle:Entry')
134 ->findByUrlAndUserId($importedEntry['url'], $this->user->getId());
135 103
136 if (false !== $existingEntry) { 104 $entry = new Entry($this->user);
137 ++$this->skippedEntries; 105 $entry->setUrl($data['url']);
138 continue; 106 $entry->setTitle($data['title']);
139 }
140 107
141 $data = $this->prepareEntry($importedEntry, $this->markAsRead); 108 // update entry with content (in case fetching failed, the given entry will be return)
109 $entry = $this->fetchContent($entry, $data['url'], $data);
142 110
143 $entry = $this->fetchContent( 111 if (array_key_exists('tags', $data)) {
144 new Entry($this->user), 112 $this->contentProxy->assignTagsToEntry(
145 $importedEntry['url'], 113 $entry,
146 $data 114 $data['tags']
147 ); 115 );
116 }
148 117
149 // jump to next entry in case of problem while getting content 118 if (isset($importedEntry['preview_picture'])) {
150 if (false === $entry) { 119 $entry->setPreviewPicture($importedEntry['preview_picture']);
151 ++$this->skippedEntries;
152 continue;
153 }
154
155 if (array_key_exists('tags', $data)) {
156 $this->contentProxy->assignTagsToEntry(
157 $entry,
158 $data['tags']
159 );
160 }
161
162 if (isset($importedEntry['preview_picture'])) {
163 $entry->setPreviewPicture($importedEntry['preview_picture']);
164 }
165
166 $entry->setArchived($data['is_archived']);
167 $entry->setStarred($data['is_starred']);
168
169 $this->em->persist($entry);
170 ++$this->importedEntries;
171
172 // flush every 20 entries
173 if (($i % 20) === 0) {
174 $this->em->flush();
175 }
176 ++$i;
177 } 120 }
178 121
179 $this->em->flush(); 122 $entry->setArchived($data['is_archived']);
180 $this->em->clear(); 123 $entry->setStarred($data['is_starred']);
124
125 if (!empty($data['created_at'])) {
126 $entry->setCreatedAt(new \DateTime($data['created_at']));
127 }
128
129 $this->em->persist($entry);
130 ++$this->importedEntries;
131
132 return $entry;
181 } 133 }
182 134
183 /** 135 /**
184 * This should return a cleaned array for a given entry to be given to `updateEntry`. 136 * This should return a cleaned array for a given entry to be given to `updateEntry`.
185 * 137 *
186 * @param array $entry Data from the imported file 138 * @param array $entry Data from the imported file
187 * @param bool $markAsRead Should we mark as read content?
188 * 139 *
189 * @return array 140 * @return array
190 */ 141 */
191 abstract protected function prepareEntry($entry = [], $markAsRead = false); 142 abstract protected function prepareEntry($entry = []);
192} 143}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
index 6cf3467a..4f001062 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
@@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport
31 /** 31 /**
32 * {@inheritdoc} 32 * {@inheritdoc}
33 */ 33 */
34 protected function prepareEntry($entry = [], $markAsRead = false) 34 protected function prepareEntry($entry = [])
35 { 35 {
36 $data = [ 36 $data = [
37 'title' => $entry['title'], 37 'title' => $entry['title'],
@@ -39,9 +39,10 @@ class WallabagV1Import extends WallabagImport
39 'url' => $entry['url'], 39 'url' => $entry['url'],
40 'content_type' => '', 40 'content_type' => '',
41 'language' => '', 41 'language' => '',
42 'is_archived' => $entry['is_read'] || $markAsRead, 42 'is_archived' => $entry['is_read'] || $this->markAsRead,
43 'is_starred' => $entry['is_fav'], 43 'is_starred' => $entry['is_fav'],
44 'tags' => '', 44 'tags' => '',
45 'created_at' => '',
45 ]; 46 ];
46 47
47 // force content to be refreshed in case on bad fetch in the v1 installation 48 // force content to be refreshed in case on bad fetch in the v1 installation
@@ -56,4 +57,14 @@ class WallabagV1Import extends WallabagImport
56 57
57 return $data; 58 return $data;
58 } 59 }
60
61 /**
62 * {@inheritdoc}
63 */
64 protected function setEntryAsRead(array $importedEntry)
65 {
66 $importedEntry['is_read'] = 1;
67
68 return $importedEntry;
69 }
59} 70}
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
index d0035b63..37c8ca14 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
@@ -31,12 +31,22 @@ class WallabagV2Import extends WallabagImport
31 /** 31 /**
32 * {@inheritdoc} 32 * {@inheritdoc}
33 */ 33 */
34 protected function prepareEntry($entry = [], $markAsRead = false) 34 protected function prepareEntry($entry = [])
35 { 35 {
36 return [ 36 return [
37 'html' => $entry['content'], 37 'html' => $entry['content'],
38 'content_type' => $entry['mimetype'], 38 'content_type' => $entry['mimetype'],
39 'is_archived' => ($entry['is_archived'] || $markAsRead), 39 'is_archived' => ($entry['is_archived'] || $this->markAsRead),
40 ] + $entry; 40 ] + $entry;
41 } 41 }
42
43 /**
44 * {@inheritdoc}
45 */
46 protected function setEntryAsRead(array $importedEntry)
47 {
48 $importedEntry['is_archived'] = 1;
49
50 return $importedEntry;
51 }
42} 52}