diff options
author | Paulino Michelazzo <paulino@michelazzo.com.br> | 2016-10-18 22:48:23 +0200 |
---|---|---|
committer | Paulino Michelazzo <paulino@michelazzo.com.br> | 2016-10-18 22:48:23 +0200 |
commit | 99731f0bb1f6fd2815eeb9af504ce86df927657b (patch) | |
tree | b080efc608d2bbd52b77a4a0067402007f50c5a8 /src/Wallabag/ImportBundle/Import | |
parent | 3a3c6b866b52721431bed22426d9abfcd0d2dfe0 (diff) | |
parent | 7180aaed45dce62e40620a9e4b202526ebd6a3bb (diff) | |
download | wallabag-99731f0bb1f6fd2815eeb9af504ce86df927657b.tar.gz wallabag-99731f0bb1f6fd2815eeb9af504ce86df927657b.tar.zst wallabag-99731f0bb1f6fd2815eeb9af504ce86df927657b.zip |
Merge remote-tracking branch 'wallabag/master'
Diffstat (limited to 'src/Wallabag/ImportBundle/Import')
-rw-r--r-- | src/Wallabag/ImportBundle/Import/AbstractImport.php | 148 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/BrowserImport.php | 207 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/ChromeImport.php | 53 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/FirefoxImport.php | 53 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/InstapaperImport.php | 140 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/PocketImport.php | 219 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/ReadabilityImport.php | 134 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/WallabagImport.php | 134 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/WallabagV1Import.php | 15 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/WallabagV2Import.php | 14 |
10 files changed, 905 insertions, 212 deletions
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 14377a35..764b390a 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php | |||
@@ -7,12 +7,21 @@ use Psr\Log\NullLogger; | |||
7 | use Doctrine\ORM\EntityManager; | 7 | use Doctrine\ORM\EntityManager; |
8 | use Wallabag\CoreBundle\Helper\ContentProxy; | 8 | use Wallabag\CoreBundle\Helper\ContentProxy; |
9 | use Wallabag\CoreBundle\Entity\Entry; | 9 | use Wallabag\CoreBundle\Entity\Entry; |
10 | use Wallabag\CoreBundle\Entity\Tag; | ||
11 | use Wallabag\UserBundle\Entity\User; | ||
12 | use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; | ||
10 | 13 | ||
11 | abstract class AbstractImport implements ImportInterface | 14 | abstract class AbstractImport implements ImportInterface |
12 | { | 15 | { |
13 | protected $em; | 16 | protected $em; |
14 | protected $logger; | 17 | protected $logger; |
15 | protected $contentProxy; | 18 | protected $contentProxy; |
19 | protected $producer; | ||
20 | protected $user; | ||
21 | protected $markAsRead; | ||
22 | protected $skippedEntries = 0; | ||
23 | protected $importedEntries = 0; | ||
24 | protected $queuedEntries = 0; | ||
16 | 25 | ||
17 | public function __construct(EntityManager $em, ContentProxy $contentProxy) | 26 | public function __construct(EntityManager $em, ContentProxy $contentProxy) |
18 | { | 27 | { |
@@ -27,21 +36,154 @@ abstract class AbstractImport implements ImportInterface | |||
27 | } | 36 | } |
28 | 37 | ||
29 | /** | 38 | /** |
39 | * Set RabbitMQ/Redis Producer to send each entry to a queue. | ||
40 | * This method should be called when user has enabled RabbitMQ. | ||
41 | * | ||
42 | * @param ProducerInterface $producer | ||
43 | */ | ||
44 | public function setProducer(ProducerInterface $producer) | ||
45 | { | ||
46 | $this->producer = $producer; | ||
47 | } | ||
48 | |||
49 | /** | ||
50 | * Set current user. | ||
51 | * Could the current *connected* user or one retrieve by the consumer. | ||
52 | * | ||
53 | * @param User $user | ||
54 | */ | ||
55 | public function setUser(User $user) | ||
56 | { | ||
57 | $this->user = $user; | ||
58 | } | ||
59 | |||
60 | /** | ||
61 | * Set whether articles must be all marked as read. | ||
62 | * | ||
63 | * @param bool $markAsRead | ||
64 | */ | ||
65 | public function setMarkAsRead($markAsRead) | ||
66 | { | ||
67 | $this->markAsRead = $markAsRead; | ||
68 | |||
69 | return $this; | ||
70 | } | ||
71 | |||
72 | /** | ||
73 | * Get whether articles must be all marked as read. | ||
74 | */ | ||
75 | public function getMarkAsRead() | ||
76 | { | ||
77 | return $this->markAsRead; | ||
78 | } | ||
79 | |||
80 | /** | ||
30 | * Fetch content from the ContentProxy (using graby). | 81 | * Fetch content from the ContentProxy (using graby). |
31 | * If it fails return false instead of the updated entry. | 82 | * If it fails return the given entry to be saved in all case (to avoid user to loose the content). |
32 | * | 83 | * |
33 | * @param Entry $entry Entry to update | 84 | * @param Entry $entry Entry to update |
34 | * @param string $url Url to grab content for | 85 | * @param string $url Url to grab content for |
35 | * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url | 86 | * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url |
36 | * | 87 | * |
37 | * @return Entry|false | 88 | * @return Entry |
38 | */ | 89 | */ |
39 | protected function fetchContent(Entry $entry, $url, array $content = []) | 90 | protected function fetchContent(Entry $entry, $url, array $content = []) |
40 | { | 91 | { |
41 | try { | 92 | try { |
42 | return $this->contentProxy->updateEntry($entry, $url, $content); | 93 | return $this->contentProxy->updateEntry($entry, $url, $content); |
43 | } catch (\Exception $e) { | 94 | } catch (\Exception $e) { |
44 | return false; | 95 | return $entry; |
45 | } | 96 | } |
46 | } | 97 | } |
98 | |||
99 | /** | ||
100 | * Parse and insert all given entries. | ||
101 | * | ||
102 | * @param $entries | ||
103 | */ | ||
104 | protected function parseEntries($entries) | ||
105 | { | ||
106 | $i = 1; | ||
107 | |||
108 | foreach ($entries as $importedEntry) { | ||
109 | if ($this->markAsRead) { | ||
110 | $importedEntry = $this->setEntryAsRead($importedEntry); | ||
111 | } | ||
112 | |||
113 | $entry = $this->parseEntry($importedEntry); | ||
114 | |||
115 | if (null === $entry) { | ||
116 | continue; | ||
117 | } | ||
118 | |||
119 | // flush every 20 entries | ||
120 | if (($i % 20) === 0) { | ||
121 | $this->em->flush(); | ||
122 | |||
123 | // clear only affected entities | ||
124 | $this->em->clear(Entry::class); | ||
125 | $this->em->clear(Tag::class); | ||
126 | } | ||
127 | ++$i; | ||
128 | } | ||
129 | |||
130 | $this->em->flush(); | ||
131 | } | ||
132 | |||
133 | /** | ||
134 | * Parse entries and send them to the queue. | ||
135 | * It should just be a simple loop on all item, no call to the database should be done | ||
136 | * to speedup queuing. | ||
137 | * | ||
138 | * Faster parse entries for Producer. | ||
139 | * We don't care to make check at this time. They'll be done by the consumer. | ||
140 | * | ||
141 | * @param array $entries | ||
142 | */ | ||
143 | protected function parseEntriesForProducer(array $entries) | ||
144 | { | ||
145 | foreach ($entries as $importedEntry) { | ||
146 | // set userId for the producer (it won't know which user is connected) | ||
147 | $importedEntry['userId'] = $this->user->getId(); | ||
148 | |||
149 | if ($this->markAsRead) { | ||
150 | $importedEntry = $this->setEntryAsRead($importedEntry); | ||
151 | } | ||
152 | |||
153 | ++$this->queuedEntries; | ||
154 | |||
155 | $this->producer->publish(json_encode($importedEntry)); | ||
156 | } | ||
157 | } | ||
158 | |||
159 | /** | ||
160 | * {@inheritdoc} | ||
161 | */ | ||
162 | public function getSummary() | ||
163 | { | ||
164 | return [ | ||
165 | 'skipped' => $this->skippedEntries, | ||
166 | 'imported' => $this->importedEntries, | ||
167 | 'queued' => $this->queuedEntries, | ||
168 | ]; | ||
169 | } | ||
170 | |||
171 | /** | ||
172 | * Parse one entry. | ||
173 | * | ||
174 | * @param array $importedEntry | ||
175 | * | ||
176 | * @return Entry | ||
177 | */ | ||
178 | abstract public function parseEntry(array $importedEntry); | ||
179 | |||
180 | /** | ||
181 | * Set current imported entry to archived / read. | ||
182 | * Implementation is different accross all imports. | ||
183 | * | ||
184 | * @param array $importedEntry | ||
185 | * | ||
186 | * @return array | ||
187 | */ | ||
188 | abstract protected function setEntryAsRead(array $importedEntry); | ||
47 | } | 189 | } |
diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php new file mode 100644 index 00000000..9d75685b --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php | |||
@@ -0,0 +1,207 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Wallabag\CoreBundle\Entity\Entry; | ||
6 | use Wallabag\UserBundle\Entity\User; | ||
7 | use Wallabag\CoreBundle\Helper\ContentProxy; | ||
8 | |||
9 | abstract class BrowserImport extends AbstractImport | ||
10 | { | ||
11 | protected $filepath; | ||
12 | |||
13 | /** | ||
14 | * {@inheritdoc} | ||
15 | */ | ||
16 | abstract public function getName(); | ||
17 | |||
18 | /** | ||
19 | * {@inheritdoc} | ||
20 | */ | ||
21 | abstract public function getUrl(); | ||
22 | |||
23 | /** | ||
24 | * {@inheritdoc} | ||
25 | */ | ||
26 | abstract public function getDescription(); | ||
27 | |||
28 | /** | ||
29 | * {@inheritdoc} | ||
30 | */ | ||
31 | public function import() | ||
32 | { | ||
33 | if (!$this->user) { | ||
34 | $this->logger->error('Wallabag Browser Import: user is not defined'); | ||
35 | |||
36 | return false; | ||
37 | } | ||
38 | |||
39 | if (!file_exists($this->filepath) || !is_readable($this->filepath)) { | ||
40 | $this->logger->error('Wallabag Browser Import: unable to read file', ['filepath' => $this->filepath]); | ||
41 | |||
42 | return false; | ||
43 | } | ||
44 | |||
45 | $data = json_decode(file_get_contents($this->filepath), true); | ||
46 | |||
47 | if (empty($data)) { | ||
48 | $this->logger->error('Wallabag Browser: no entries in imported file'); | ||
49 | |||
50 | return false; | ||
51 | } | ||
52 | |||
53 | if ($this->producer) { | ||
54 | $this->parseEntriesForProducer($data); | ||
55 | |||
56 | return true; | ||
57 | } | ||
58 | |||
59 | $this->parseEntries($data); | ||
60 | |||
61 | return true; | ||
62 | } | ||
63 | |||
64 | /** | ||
65 | * Set file path to the json file. | ||
66 | * | ||
67 | * @param string $filepath | ||
68 | */ | ||
69 | public function setFilepath($filepath) | ||
70 | { | ||
71 | $this->filepath = $filepath; | ||
72 | |||
73 | return $this; | ||
74 | } | ||
75 | |||
76 | /** | ||
77 | * Parse and insert all given entries. | ||
78 | * | ||
79 | * @param $entries | ||
80 | */ | ||
81 | protected function parseEntries($entries) | ||
82 | { | ||
83 | $i = 1; | ||
84 | |||
85 | foreach ($entries as $importedEntry) { | ||
86 | if ((array) $importedEntry !== $importedEntry) { | ||
87 | continue; | ||
88 | } | ||
89 | |||
90 | $entry = $this->parseEntry($importedEntry); | ||
91 | |||
92 | if (null === $entry) { | ||
93 | continue; | ||
94 | } | ||
95 | |||
96 | // flush every 20 entries | ||
97 | if (($i % 20) === 0) { | ||
98 | $this->em->flush(); | ||
99 | } | ||
100 | ++$i; | ||
101 | } | ||
102 | |||
103 | $this->em->flush(); | ||
104 | } | ||
105 | |||
106 | /** | ||
107 | * Parse entries and send them to the queue. | ||
108 | * It should just be a simple loop on all item, no call to the database should be done | ||
109 | * to speedup queuing. | ||
110 | * | ||
111 | * Faster parse entries for Producer. | ||
112 | * We don't care to make check at this time. They'll be done by the consumer. | ||
113 | * | ||
114 | * @param array $entries | ||
115 | */ | ||
116 | protected function parseEntriesForProducer(array $entries) | ||
117 | { | ||
118 | foreach ($entries as $importedEntry) { | ||
119 | if ((array) $importedEntry !== $importedEntry) { | ||
120 | continue; | ||
121 | } | ||
122 | |||
123 | // set userId for the producer (it won't know which user is connected) | ||
124 | $importedEntry['userId'] = $this->user->getId(); | ||
125 | |||
126 | if ($this->markAsRead) { | ||
127 | $importedEntry = $this->setEntryAsRead($importedEntry); | ||
128 | } | ||
129 | |||
130 | ++$this->queuedEntries; | ||
131 | |||
132 | $this->producer->publish(json_encode($importedEntry)); | ||
133 | } | ||
134 | } | ||
135 | |||
136 | /** | ||
137 | * {@inheritdoc} | ||
138 | */ | ||
139 | public function parseEntry(array $importedEntry) | ||
140 | { | ||
141 | if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { | ||
142 | $this->parseEntries($importedEntry); | ||
143 | |||
144 | return; | ||
145 | } | ||
146 | |||
147 | if (array_key_exists('children', $importedEntry)) { | ||
148 | $this->parseEntries($importedEntry['children']); | ||
149 | |||
150 | return; | ||
151 | } | ||
152 | |||
153 | if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) { | ||
154 | return; | ||
155 | } | ||
156 | |||
157 | $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url']; | ||
158 | |||
159 | $existingEntry = $this->em | ||
160 | ->getRepository('WallabagCoreBundle:Entry') | ||
161 | ->findByUrlAndUserId($url, $this->user->getId()); | ||
162 | |||
163 | if (false !== $existingEntry) { | ||
164 | ++$this->skippedEntries; | ||
165 | |||
166 | return; | ||
167 | } | ||
168 | |||
169 | $data = $this->prepareEntry($importedEntry); | ||
170 | |||
171 | $entry = new Entry($this->user); | ||
172 | $entry->setUrl($data['url']); | ||
173 | $entry->setTitle($data['title']); | ||
174 | |||
175 | // update entry with content (in case fetching failed, the given entry will be return) | ||
176 | $entry = $this->fetchContent($entry, $data['url'], $data); | ||
177 | |||
178 | if (array_key_exists('tags', $data)) { | ||
179 | $this->contentProxy->assignTagsToEntry( | ||
180 | $entry, | ||
181 | $data['tags'] | ||
182 | ); | ||
183 | } | ||
184 | |||
185 | $entry->setArchived($data['is_archived']); | ||
186 | |||
187 | if (!empty($data['created_at'])) { | ||
188 | $dt = new \DateTime(); | ||
189 | $entry->setCreatedAt($dt->setTimestamp($data['created_at'])); | ||
190 | } | ||
191 | |||
192 | $this->em->persist($entry); | ||
193 | ++$this->importedEntries; | ||
194 | |||
195 | return $entry; | ||
196 | } | ||
197 | |||
198 | /** | ||
199 | * {@inheritdoc} | ||
200 | */ | ||
201 | protected function setEntryAsRead(array $importedEntry) | ||
202 | { | ||
203 | $importedEntry['is_archived'] = 1; | ||
204 | |||
205 | return $importedEntry; | ||
206 | } | ||
207 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php new file mode 100644 index 00000000..d7620bcb --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php | |||
@@ -0,0 +1,53 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | class ChromeImport extends BrowserImport | ||
6 | { | ||
7 | protected $filepath; | ||
8 | |||
9 | /** | ||
10 | * {@inheritdoc} | ||
11 | */ | ||
12 | public function getName() | ||
13 | { | ||
14 | return 'Chrome'; | ||
15 | } | ||
16 | |||
17 | /** | ||
18 | * {@inheritdoc} | ||
19 | */ | ||
20 | public function getUrl() | ||
21 | { | ||
22 | return 'import_chrome'; | ||
23 | } | ||
24 | |||
25 | /** | ||
26 | * {@inheritdoc} | ||
27 | */ | ||
28 | public function getDescription() | ||
29 | { | ||
30 | return 'import.chrome.description'; | ||
31 | } | ||
32 | |||
33 | /** | ||
34 | * {@inheritdoc} | ||
35 | */ | ||
36 | protected function prepareEntry(array $entry = []) | ||
37 | { | ||
38 | $data = [ | ||
39 | 'title' => $entry['name'], | ||
40 | 'html' => '', | ||
41 | 'url' => $entry['url'], | ||
42 | 'is_archived' => $this->markAsRead, | ||
43 | 'tags' => '', | ||
44 | 'created_at' => substr($entry['date_added'], 0, 10), | ||
45 | ]; | ||
46 | |||
47 | if (array_key_exists('tags', $entry) && $entry['tags'] != '') { | ||
48 | $data['tags'] = $entry['tags']; | ||
49 | } | ||
50 | |||
51 | return $data; | ||
52 | } | ||
53 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php new file mode 100644 index 00000000..e010f5a4 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php | |||
@@ -0,0 +1,53 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | class FirefoxImport extends BrowserImport | ||
6 | { | ||
7 | protected $filepath; | ||
8 | |||
9 | /** | ||
10 | * {@inheritdoc} | ||
11 | */ | ||
12 | public function getName() | ||
13 | { | ||
14 | return 'Firefox'; | ||
15 | } | ||
16 | |||
17 | /** | ||
18 | * {@inheritdoc} | ||
19 | */ | ||
20 | public function getUrl() | ||
21 | { | ||
22 | return 'import_firefox'; | ||
23 | } | ||
24 | |||
25 | /** | ||
26 | * {@inheritdoc} | ||
27 | */ | ||
28 | public function getDescription() | ||
29 | { | ||
30 | return 'import.firefox.description'; | ||
31 | } | ||
32 | |||
33 | /** | ||
34 | * {@inheritdoc} | ||
35 | */ | ||
36 | protected function prepareEntry(array $entry = []) | ||
37 | { | ||
38 | $data = [ | ||
39 | 'title' => $entry['title'], | ||
40 | 'html' => '', | ||
41 | 'url' => $entry['uri'], | ||
42 | 'is_archived' => $this->markAsRead, | ||
43 | 'tags' => '', | ||
44 | 'created_at' => substr($entry['dateAdded'], 0, 10), | ||
45 | ]; | ||
46 | |||
47 | if (array_key_exists('tags', $entry) && $entry['tags'] != '') { | ||
48 | $data['tags'] = $entry['tags']; | ||
49 | } | ||
50 | |||
51 | return $data; | ||
52 | } | ||
53 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php new file mode 100644 index 00000000..cf4c785c --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php | |||
@@ -0,0 +1,140 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Wallabag\CoreBundle\Entity\Entry; | ||
6 | |||
7 | class InstapaperImport extends AbstractImport | ||
8 | { | ||
9 | private $filepath; | ||
10 | |||
11 | /** | ||
12 | * {@inheritdoc} | ||
13 | */ | ||
14 | public function getName() | ||
15 | { | ||
16 | return 'Instapaper'; | ||
17 | } | ||
18 | |||
19 | /** | ||
20 | * {@inheritdoc} | ||
21 | */ | ||
22 | public function getUrl() | ||
23 | { | ||
24 | return 'import_instapaper'; | ||
25 | } | ||
26 | |||
27 | /** | ||
28 | * {@inheritdoc} | ||
29 | */ | ||
30 | public function getDescription() | ||
31 | { | ||
32 | return 'import.instapaper.description'; | ||
33 | } | ||
34 | |||
35 | /** | ||
36 | * Set file path to the json file. | ||
37 | * | ||
38 | * @param string $filepath | ||
39 | */ | ||
40 | public function setFilepath($filepath) | ||
41 | { | ||
42 | $this->filepath = $filepath; | ||
43 | |||
44 | return $this; | ||
45 | } | ||
46 | |||
47 | /** | ||
48 | * {@inheritdoc} | ||
49 | */ | ||
50 | public function import() | ||
51 | { | ||
52 | if (!$this->user) { | ||
53 | $this->logger->error('InstapaperImport: user is not defined'); | ||
54 | |||
55 | return false; | ||
56 | } | ||
57 | |||
58 | if (!file_exists($this->filepath) || !is_readable($this->filepath)) { | ||
59 | $this->logger->error('InstapaperImport: unable to read file', ['filepath' => $this->filepath]); | ||
60 | |||
61 | return false; | ||
62 | } | ||
63 | |||
64 | $entries = []; | ||
65 | $handle = fopen($this->filepath, 'r'); | ||
66 | while (($data = fgetcsv($handle, 10240)) !== false) { | ||
67 | if ('URL' === $data[0]) { | ||
68 | continue; | ||
69 | } | ||
70 | |||
71 | $entries[] = [ | ||
72 | 'url' => $data[0], | ||
73 | 'title' => $data[1], | ||
74 | 'status' => $data[3], | ||
75 | 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred', | ||
76 | 'is_starred' => $data[3] === 'Starred', | ||
77 | 'content_type' => '', | ||
78 | 'language' => '', | ||
79 | ]; | ||
80 | } | ||
81 | fclose($handle); | ||
82 | |||
83 | if (empty($entries)) { | ||
84 | $this->logger->error('InstapaperImport: no entries in imported file'); | ||
85 | |||
86 | return false; | ||
87 | } | ||
88 | |||
89 | if ($this->producer) { | ||
90 | $this->parseEntriesForProducer($entries); | ||
91 | |||
92 | return true; | ||
93 | } | ||
94 | |||
95 | $this->parseEntries($entries); | ||
96 | |||
97 | return true; | ||
98 | } | ||
99 | |||
100 | /** | ||
101 | * {@inheritdoc} | ||
102 | */ | ||
103 | public function parseEntry(array $importedEntry) | ||
104 | { | ||
105 | $existingEntry = $this->em | ||
106 | ->getRepository('WallabagCoreBundle:Entry') | ||
107 | ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); | ||
108 | |||
109 | if (false !== $existingEntry) { | ||
110 | ++$this->skippedEntries; | ||
111 | |||
112 | return; | ||
113 | } | ||
114 | |||
115 | $entry = new Entry($this->user); | ||
116 | $entry->setUrl($importedEntry['url']); | ||
117 | $entry->setTitle($importedEntry['title']); | ||
118 | |||
119 | // update entry with content (in case fetching failed, the given entry will be return) | ||
120 | $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry); | ||
121 | |||
122 | $entry->setArchived($importedEntry['is_archived']); | ||
123 | $entry->setStarred($importedEntry['is_starred']); | ||
124 | |||
125 | $this->em->persist($entry); | ||
126 | ++$this->importedEntries; | ||
127 | |||
128 | return $entry; | ||
129 | } | ||
130 | |||
131 | /** | ||
132 | * {@inheritdoc} | ||
133 | */ | ||
134 | protected function setEntryAsRead(array $importedEntry) | ||
135 | { | ||
136 | $importedEntry['is_archived'] = 1; | ||
137 | |||
138 | return $importedEntry; | ||
139 | } | ||
140 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 798cfdae..327e2500 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php | |||
@@ -6,31 +6,34 @@ use Psr\Log\NullLogger; | |||
6 | use Doctrine\ORM\EntityManager; | 6 | use Doctrine\ORM\EntityManager; |
7 | use GuzzleHttp\Client; | 7 | use GuzzleHttp\Client; |
8 | use GuzzleHttp\Exception\RequestException; | 8 | use GuzzleHttp\Exception\RequestException; |
9 | use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; | ||
10 | use Wallabag\CoreBundle\Entity\Entry; | 9 | use Wallabag\CoreBundle\Entity\Entry; |
11 | use Wallabag\CoreBundle\Helper\ContentProxy; | 10 | use Wallabag\CoreBundle\Helper\ContentProxy; |
12 | use Craue\ConfigBundle\Util\Config; | ||
13 | 11 | ||
14 | class PocketImport extends AbstractImport | 12 | class PocketImport extends AbstractImport |
15 | { | 13 | { |
16 | private $user; | ||
17 | private $client; | 14 | private $client; |
18 | private $consumerKey; | 15 | private $accessToken; |
19 | private $skippedEntries = 0; | ||
20 | private $importedEntries = 0; | ||
21 | private $markAsRead; | ||
22 | protected $accessToken; | ||
23 | 16 | ||
24 | public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) | 17 | const NB_ELEMENTS = 5000; |
18 | |||
19 | public function __construct(EntityManager $em, ContentProxy $contentProxy) | ||
25 | { | 20 | { |
26 | $this->user = $tokenStorage->getToken()->getUser(); | ||
27 | $this->em = $em; | 21 | $this->em = $em; |
28 | $this->contentProxy = $contentProxy; | 22 | $this->contentProxy = $contentProxy; |
29 | $this->consumerKey = $craueConfig->get('pocket_consumer_key'); | ||
30 | $this->logger = new NullLogger(); | 23 | $this->logger = new NullLogger(); |
31 | } | 24 | } |
32 | 25 | ||
33 | /** | 26 | /** |
27 | * Only used for test purpose. | ||
28 | * | ||
29 | * @return string | ||
30 | */ | ||
31 | public function getAccessToken() | ||
32 | { | ||
33 | return $this->accessToken; | ||
34 | } | ||
35 | |||
36 | /** | ||
34 | * {@inheritdoc} | 37 | * {@inheritdoc} |
35 | */ | 38 | */ |
36 | public function getName() | 39 | public function getName() |
@@ -66,7 +69,7 @@ class PocketImport extends AbstractImport | |||
66 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', | 69 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', |
67 | [ | 70 | [ |
68 | 'body' => json_encode([ | 71 | 'body' => json_encode([ |
69 | 'consumer_key' => $this->consumerKey, | 72 | 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), |
70 | 'redirect_uri' => $redirectUri, | 73 | 'redirect_uri' => $redirectUri, |
71 | ]), | 74 | ]), |
72 | ] | 75 | ] |
@@ -96,7 +99,7 @@ class PocketImport extends AbstractImport | |||
96 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', | 99 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', |
97 | [ | 100 | [ |
98 | 'body' => json_encode([ | 101 | 'body' => json_encode([ |
99 | 'consumer_key' => $this->consumerKey, | 102 | 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), |
100 | 'code' => $code, | 103 | 'code' => $code, |
101 | ]), | 104 | ]), |
102 | ] | 105 | ] |
@@ -116,38 +119,22 @@ class PocketImport extends AbstractImport | |||
116 | } | 119 | } |
117 | 120 | ||
118 | /** | 121 | /** |
119 | * Set whether articles must be all marked as read. | ||
120 | * | ||
121 | * @param bool $markAsRead | ||
122 | */ | ||
123 | public function setMarkAsRead($markAsRead) | ||
124 | { | ||
125 | $this->markAsRead = $markAsRead; | ||
126 | |||
127 | return $this; | ||
128 | } | ||
129 | |||
130 | /** | ||
131 | * Get whether articles must be all marked as read. | ||
132 | */ | ||
133 | public function getMarkAsRead() | ||
134 | { | ||
135 | return $this->markAsRead; | ||
136 | } | ||
137 | |||
138 | /** | ||
139 | * {@inheritdoc} | 122 | * {@inheritdoc} |
140 | */ | 123 | */ |
141 | public function import() | 124 | public function import($offset = 0) |
142 | { | 125 | { |
126 | static $run = 0; | ||
127 | |||
143 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', | 128 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', |
144 | [ | 129 | [ |
145 | 'body' => json_encode([ | 130 | 'body' => json_encode([ |
146 | 'consumer_key' => $this->consumerKey, | 131 | 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), |
147 | 'access_token' => $this->accessToken, | 132 | 'access_token' => $this->accessToken, |
148 | 'detailType' => 'complete', | 133 | 'detailType' => 'complete', |
149 | 'state' => 'all', | 134 | 'state' => 'all', |
150 | 'sort' => 'oldest', | 135 | 'sort' => 'newest', |
136 | 'count' => self::NB_ELEMENTS, | ||
137 | 'offset' => $offset, | ||
151 | ]), | 138 | ]), |
152 | ] | 139 | ] |
153 | ); | 140 | ); |
@@ -162,20 +149,24 @@ class PocketImport extends AbstractImport | |||
162 | 149 | ||
163 | $entries = $response->json(); | 150 | $entries = $response->json(); |
164 | 151 | ||
165 | $this->parseEntries($entries['list']); | 152 | if ($this->producer) { |
153 | $this->parseEntriesForProducer($entries['list']); | ||
154 | } else { | ||
155 | $this->parseEntries($entries['list']); | ||
156 | } | ||
166 | 157 | ||
167 | return true; | 158 | // if we retrieve exactly the amount of items requested it means we can get more |
168 | } | 159 | // re-call import and offset item by the amount previous received: |
160 | // - first call get 5k offset 0 | ||
161 | // - second call get 5k offset 5k | ||
162 | // - and so on | ||
163 | if (count($entries['list']) === self::NB_ELEMENTS) { | ||
164 | ++$run; | ||
169 | 165 | ||
170 | /** | 166 | return $this->import(self::NB_ELEMENTS * $run); |
171 | * {@inheritdoc} | 167 | } |
172 | */ | 168 | |
173 | public function getSummary() | 169 | return true; |
174 | { | ||
175 | return [ | ||
176 | 'skipped' => $this->skippedEntries, | ||
177 | 'imported' => $this->importedEntries, | ||
178 | ]; | ||
179 | } | 170 | } |
180 | 171 | ||
181 | /** | 172 | /** |
@@ -189,77 +180,75 @@ class PocketImport extends AbstractImport | |||
189 | } | 180 | } |
190 | 181 | ||
191 | /** | 182 | /** |
192 | * @see https://getpocket.com/developer/docs/v3/retrieve | 183 | * {@inheritdoc} |
193 | * | 184 | * |
194 | * @param $entries | 185 | * @see https://getpocket.com/developer/docs/v3/retrieve |
195 | */ | 186 | */ |
196 | private function parseEntries($entries) | 187 | public function parseEntry(array $importedEntry) |
197 | { | 188 | { |
198 | $i = 1; | 189 | $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; |
199 | 190 | ||
200 | foreach ($entries as $pocketEntry) { | 191 | $existingEntry = $this->em |
201 | $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; | 192 | ->getRepository('WallabagCoreBundle:Entry') |
202 | 193 | ->findByUrlAndUserId($url, $this->user->getId()); | |
203 | $existingEntry = $this->em | 194 | |
204 | ->getRepository('WallabagCoreBundle:Entry') | 195 | if (false !== $existingEntry) { |
205 | ->findByUrlAndUserId($url, $this->user->getId()); | 196 | ++$this->skippedEntries; |
206 | 197 | ||
207 | if (false !== $existingEntry) { | 198 | return; |
208 | ++$this->skippedEntries; | 199 | } |
209 | continue; | 200 | |
210 | } | 201 | $entry = new Entry($this->user); |
211 | 202 | $entry->setUrl($url); | |
212 | $entry = new Entry($this->user); | 203 | |
213 | $entry = $this->fetchContent($entry, $url); | 204 | // update entry with content (in case fetching failed, the given entry will be return) |
214 | 205 | $entry = $this->fetchContent($entry, $url); | |
215 | // jump to next entry in case of problem while getting content | 206 | |
216 | if (false === $entry) { | 207 | // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted |
217 | ++$this->skippedEntries; | 208 | $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead); |
218 | continue; | 209 | |
219 | } | 210 | // 0 or 1 - 1 If the item is starred |
220 | 211 | $entry->setStarred($importedEntry['favorite'] == 1); | |
221 | // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted | 212 | |
222 | if ($pocketEntry['status'] == 1 || $this->markAsRead) { | 213 | $title = 'Untitled'; |
223 | $entry->setArchived(true); | 214 | if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') { |
224 | } | 215 | $title = $importedEntry['resolved_title']; |
225 | 216 | } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') { | |
226 | // 0 or 1 - 1 If the item is starred | 217 | $title = $importedEntry['given_title']; |
227 | if ($pocketEntry['favorite'] == 1) { | 218 | } |
228 | $entry->setStarred(true); | 219 | |
229 | } | 220 | $entry->setTitle($title); |
230 | 221 | ||
231 | $title = 'Untitled'; | 222 | // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image |
232 | if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { | 223 | if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { |
233 | $title = $pocketEntry['resolved_title']; | 224 | $entry->setPreviewPicture($importedEntry['images'][1]['src']); |
234 | } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { | 225 | } |
235 | $title = $pocketEntry['given_title']; | 226 | |
236 | } | 227 | if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { |
237 | 228 | $this->contentProxy->assignTagsToEntry( | |
238 | $entry->setTitle($title); | 229 | $entry, |
239 | 230 | array_keys($importedEntry['tags']), | |
240 | // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image | 231 | $this->em->getUnitOfWork()->getScheduledEntityInsertions() |
241 | if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { | 232 | ); |
242 | $entry->setPreviewPicture($pocketEntry['images'][1]['src']); | ||
243 | } | ||
244 | |||
245 | if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { | ||
246 | $this->contentProxy->assignTagsToEntry( | ||
247 | $entry, | ||
248 | array_keys($pocketEntry['tags']) | ||
249 | ); | ||
250 | } | ||
251 | |||
252 | $this->em->persist($entry); | ||
253 | ++$this->importedEntries; | ||
254 | |||
255 | // flush every 20 entries | ||
256 | if (($i % 20) === 0) { | ||
257 | $this->em->flush(); | ||
258 | $this->em->clear($entry); | ||
259 | } | ||
260 | ++$i; | ||
261 | } | 233 | } |
262 | 234 | ||
263 | $this->em->flush(); | 235 | if (!empty($importedEntry['time_added'])) { |
236 | $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added'])); | ||
237 | } | ||
238 | |||
239 | $this->em->persist($entry); | ||
240 | ++$this->importedEntries; | ||
241 | |||
242 | return $entry; | ||
243 | } | ||
244 | |||
245 | /** | ||
246 | * {@inheritdoc} | ||
247 | */ | ||
248 | protected function setEntryAsRead(array $importedEntry) | ||
249 | { | ||
250 | $importedEntry['status'] = '1'; | ||
251 | |||
252 | return $importedEntry; | ||
264 | } | 253 | } |
265 | } | 254 | } |
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php new file mode 100644 index 00000000..b8c0f777 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php | |||
@@ -0,0 +1,134 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Wallabag\CoreBundle\Entity\Entry; | ||
6 | |||
7 | class ReadabilityImport extends AbstractImport | ||
8 | { | ||
9 | private $filepath; | ||
10 | |||
11 | /** | ||
12 | * {@inheritdoc} | ||
13 | */ | ||
14 | public function getName() | ||
15 | { | ||
16 | return 'Readability'; | ||
17 | } | ||
18 | |||
19 | /** | ||
20 | * {@inheritdoc} | ||
21 | */ | ||
22 | public function getUrl() | ||
23 | { | ||
24 | return 'import_readability'; | ||
25 | } | ||
26 | |||
27 | /** | ||
28 | * {@inheritdoc} | ||
29 | */ | ||
30 | public function getDescription() | ||
31 | { | ||
32 | return 'import.readability.description'; | ||
33 | } | ||
34 | |||
35 | /** | ||
36 | * Set file path to the json file. | ||
37 | * | ||
38 | * @param string $filepath | ||
39 | */ | ||
40 | public function setFilepath($filepath) | ||
41 | { | ||
42 | $this->filepath = $filepath; | ||
43 | |||
44 | return $this; | ||
45 | } | ||
46 | |||
47 | /** | ||
48 | * {@inheritdoc} | ||
49 | */ | ||
50 | public function import() | ||
51 | { | ||
52 | if (!$this->user) { | ||
53 | $this->logger->error('ReadabilityImport: user is not defined'); | ||
54 | |||
55 | return false; | ||
56 | } | ||
57 | |||
58 | if (!file_exists($this->filepath) || !is_readable($this->filepath)) { | ||
59 | $this->logger->error('ReadabilityImport: unable to read file', ['filepath' => $this->filepath]); | ||
60 | |||
61 | return false; | ||
62 | } | ||
63 | |||
64 | $data = json_decode(file_get_contents($this->filepath), true); | ||
65 | |||
66 | if (empty($data) || empty($data['bookmarks'])) { | ||
67 | $this->logger->error('ReadabilityImport: no entries in imported file'); | ||
68 | |||
69 | return false; | ||
70 | } | ||
71 | |||
72 | if ($this->producer) { | ||
73 | $this->parseEntriesForProducer($data['bookmarks']); | ||
74 | |||
75 | return true; | ||
76 | } | ||
77 | |||
78 | $this->parseEntries($data['bookmarks']); | ||
79 | |||
80 | return true; | ||
81 | } | ||
82 | |||
83 | /** | ||
84 | * {@inheritdoc} | ||
85 | */ | ||
86 | public function parseEntry(array $importedEntry) | ||
87 | { | ||
88 | $existingEntry = $this->em | ||
89 | ->getRepository('WallabagCoreBundle:Entry') | ||
90 | ->findByUrlAndUserId($importedEntry['article__url'], $this->user->getId()); | ||
91 | |||
92 | if (false !== $existingEntry) { | ||
93 | ++$this->skippedEntries; | ||
94 | |||
95 | return; | ||
96 | } | ||
97 | |||
98 | $data = [ | ||
99 | 'title' => $importedEntry['article__title'], | ||
100 | 'url' => $importedEntry['article__url'], | ||
101 | 'content_type' => '', | ||
102 | 'language' => '', | ||
103 | 'is_archived' => $importedEntry['archive'] || $this->markAsRead, | ||
104 | 'is_starred' => $importedEntry['favorite'], | ||
105 | 'created_at' => $importedEntry['date_added'], | ||
106 | ]; | ||
107 | |||
108 | $entry = new Entry($this->user); | ||
109 | $entry->setUrl($data['url']); | ||
110 | $entry->setTitle($data['title']); | ||
111 | |||
112 | // update entry with content (in case fetching failed, the given entry will be return) | ||
113 | $entry = $this->fetchContent($entry, $data['url'], $data); | ||
114 | |||
115 | $entry->setArchived($data['is_archived']); | ||
116 | $entry->setStarred($data['is_starred']); | ||
117 | $entry->setCreatedAt(new \DateTime($data['created_at'])); | ||
118 | |||
119 | $this->em->persist($entry); | ||
120 | ++$this->importedEntries; | ||
121 | |||
122 | return $entry; | ||
123 | } | ||
124 | |||
125 | /** | ||
126 | * {@inheritdoc} | ||
127 | */ | ||
128 | protected function setEntryAsRead(array $importedEntry) | ||
129 | { | ||
130 | $importedEntry['archive'] = 1; | ||
131 | |||
132 | return $importedEntry; | ||
133 | } | ||
134 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php index a1cc085b..702da057 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagImport.php +++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php | |||
@@ -3,15 +3,10 @@ | |||
3 | namespace Wallabag\ImportBundle\Import; | 3 | namespace Wallabag\ImportBundle\Import; |
4 | 4 | ||
5 | use Wallabag\CoreBundle\Entity\Entry; | 5 | use Wallabag\CoreBundle\Entity\Entry; |
6 | use Wallabag\UserBundle\Entity\User; | ||
7 | 6 | ||
8 | abstract class WallabagImport extends AbstractImport | 7 | abstract class WallabagImport extends AbstractImport |
9 | { | 8 | { |
10 | protected $user; | ||
11 | protected $skippedEntries = 0; | ||
12 | protected $importedEntries = 0; | ||
13 | protected $filepath; | 9 | protected $filepath; |
14 | protected $markAsRead; | ||
15 | // untitled in all languages from v1 | 10 | // untitled in all languages from v1 |
16 | protected $untitled = [ | 11 | protected $untitled = [ |
17 | 'Untitled', | 12 | 'Untitled', |
@@ -29,19 +24,6 @@ abstract class WallabagImport extends AbstractImport | |||
29 | ]; | 24 | ]; |
30 | 25 | ||
31 | /** | 26 | /** |
32 | * We define the user in a custom call because on the import command there is no logged in user. | ||
33 | * So we can't retrieve user from the `security.token_storage` service. | ||
34 | * | ||
35 | * @param User $user | ||
36 | */ | ||
37 | public function setUser(User $user) | ||
38 | { | ||
39 | $this->user = $user; | ||
40 | |||
41 | return $this; | ||
42 | } | ||
43 | |||
44 | /** | ||
45 | * {@inheritdoc} | 27 | * {@inheritdoc} |
46 | */ | 28 | */ |
47 | abstract public function getName(); | 29 | abstract public function getName(); |
@@ -76,26 +58,23 @@ abstract class WallabagImport extends AbstractImport | |||
76 | $data = json_decode(file_get_contents($this->filepath), true); | 58 | $data = json_decode(file_get_contents($this->filepath), true); |
77 | 59 | ||
78 | if (empty($data)) { | 60 | if (empty($data)) { |
61 | $this->logger->error('WallabagImport: no entries in imported file'); | ||
62 | |||
79 | return false; | 63 | return false; |
80 | } | 64 | } |
81 | 65 | ||
66 | if ($this->producer) { | ||
67 | $this->parseEntriesForProducer($data); | ||
68 | |||
69 | return true; | ||
70 | } | ||
71 | |||
82 | $this->parseEntries($data); | 72 | $this->parseEntries($data); |
83 | 73 | ||
84 | return true; | 74 | return true; |
85 | } | 75 | } |
86 | 76 | ||
87 | /** | 77 | /** |
88 | * {@inheritdoc} | ||
89 | */ | ||
90 | public function getSummary() | ||
91 | { | ||
92 | return [ | ||
93 | 'skipped' => $this->skippedEntries, | ||
94 | 'imported' => $this->importedEntries, | ||
95 | ]; | ||
96 | } | ||
97 | |||
98 | /** | ||
99 | * Set file path to the json file. | 78 | * Set file path to the json file. |
100 | * | 79 | * |
101 | * @param string $filepath | 80 | * @param string $filepath |
@@ -108,85 +87,60 @@ abstract class WallabagImport extends AbstractImport | |||
108 | } | 87 | } |
109 | 88 | ||
110 | /** | 89 | /** |
111 | * Set whether articles must be all marked as read. | 90 | * {@inheritdoc} |
112 | * | ||
113 | * @param bool $markAsRead | ||
114 | */ | 91 | */ |
115 | public function setMarkAsRead($markAsRead) | 92 | public function parseEntry(array $importedEntry) |
116 | { | 93 | { |
117 | $this->markAsRead = $markAsRead; | 94 | $existingEntry = $this->em |
95 | ->getRepository('WallabagCoreBundle:Entry') | ||
96 | ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); | ||
118 | 97 | ||
119 | return $this; | 98 | if (false !== $existingEntry) { |
120 | } | 99 | ++$this->skippedEntries; |
121 | 100 | ||
122 | /** | 101 | return; |
123 | * Parse and insert all given entries. | 102 | } |
124 | * | ||
125 | * @param $entries | ||
126 | */ | ||
127 | protected function parseEntries($entries) | ||
128 | { | ||
129 | $i = 1; | ||
130 | 103 | ||
131 | foreach ($entries as $importedEntry) { | 104 | $data = $this->prepareEntry($importedEntry); |
132 | $existingEntry = $this->em | ||
133 | ->getRepository('WallabagCoreBundle:Entry') | ||
134 | ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); | ||
135 | 105 | ||
136 | if (false !== $existingEntry) { | 106 | $entry = new Entry($this->user); |
137 | ++$this->skippedEntries; | 107 | $entry->setUrl($data['url']); |
138 | continue; | 108 | $entry->setTitle($data['title']); |
139 | } | ||
140 | 109 | ||
141 | $data = $this->prepareEntry($importedEntry, $this->markAsRead); | 110 | // update entry with content (in case fetching failed, the given entry will be return) |
111 | $entry = $this->fetchContent($entry, $data['url'], $data); | ||
142 | 112 | ||
143 | $entry = $this->fetchContent( | 113 | if (array_key_exists('tags', $data)) { |
144 | new Entry($this->user), | 114 | $this->contentProxy->assignTagsToEntry( |
145 | $importedEntry['url'], | 115 | $entry, |
146 | $data | 116 | $data['tags'], |
117 | $this->em->getUnitOfWork()->getScheduledEntityInsertions() | ||
147 | ); | 118 | ); |
119 | } | ||
148 | 120 | ||
149 | // jump to next entry in case of problem while getting content | 121 | if (isset($importedEntry['preview_picture'])) { |
150 | if (false === $entry) { | 122 | $entry->setPreviewPicture($importedEntry['preview_picture']); |
151 | ++$this->skippedEntries; | ||
152 | continue; | ||
153 | } | ||
154 | |||
155 | if (array_key_exists('tags', $data)) { | ||
156 | $this->contentProxy->assignTagsToEntry( | ||
157 | $entry, | ||
158 | $data['tags'] | ||
159 | ); | ||
160 | } | ||
161 | |||
162 | if (isset($importedEntry['preview_picture'])) { | ||
163 | $entry->setPreviewPicture($importedEntry['preview_picture']); | ||
164 | } | ||
165 | |||
166 | $entry->setArchived($data['is_archived']); | ||
167 | $entry->setStarred($data['is_starred']); | ||
168 | |||
169 | $this->em->persist($entry); | ||
170 | ++$this->importedEntries; | ||
171 | |||
172 | // flush every 20 entries | ||
173 | if (($i % 20) === 0) { | ||
174 | $this->em->flush(); | ||
175 | $this->em->clear($entry); | ||
176 | } | ||
177 | ++$i; | ||
178 | } | 123 | } |
179 | 124 | ||
180 | $this->em->flush(); | 125 | $entry->setArchived($data['is_archived']); |
126 | $entry->setStarred($data['is_starred']); | ||
127 | |||
128 | if (!empty($data['created_at'])) { | ||
129 | $entry->setCreatedAt(new \DateTime($data['created_at'])); | ||
130 | } | ||
131 | |||
132 | $this->em->persist($entry); | ||
133 | ++$this->importedEntries; | ||
134 | |||
135 | return $entry; | ||
181 | } | 136 | } |
182 | 137 | ||
183 | /** | 138 | /** |
184 | * This should return a cleaned array for a given entry to be given to `updateEntry`. | 139 | * This should return a cleaned array for a given entry to be given to `updateEntry`. |
185 | * | 140 | * |
186 | * @param array $entry Data from the imported file | 141 | * @param array $entry Data from the imported file |
187 | * @param bool $markAsRead Should we mark as read content? | ||
188 | * | 142 | * |
189 | * @return array | 143 | * @return array |
190 | */ | 144 | */ |
191 | abstract protected function prepareEntry($entry = [], $markAsRead = false); | 145 | abstract protected function prepareEntry($entry = []); |
192 | } | 146 | } |
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php index 6cf3467a..4f001062 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php | |||
@@ -31,7 +31,7 @@ class WallabagV1Import extends WallabagImport | |||
31 | /** | 31 | /** |
32 | * {@inheritdoc} | 32 | * {@inheritdoc} |
33 | */ | 33 | */ |
34 | protected function prepareEntry($entry = [], $markAsRead = false) | 34 | protected function prepareEntry($entry = []) |
35 | { | 35 | { |
36 | $data = [ | 36 | $data = [ |
37 | 'title' => $entry['title'], | 37 | 'title' => $entry['title'], |
@@ -39,9 +39,10 @@ class WallabagV1Import extends WallabagImport | |||
39 | 'url' => $entry['url'], | 39 | 'url' => $entry['url'], |
40 | 'content_type' => '', | 40 | 'content_type' => '', |
41 | 'language' => '', | 41 | 'language' => '', |
42 | 'is_archived' => $entry['is_read'] || $markAsRead, | 42 | 'is_archived' => $entry['is_read'] || $this->markAsRead, |
43 | 'is_starred' => $entry['is_fav'], | 43 | 'is_starred' => $entry['is_fav'], |
44 | 'tags' => '', | 44 | 'tags' => '', |
45 | 'created_at' => '', | ||
45 | ]; | 46 | ]; |
46 | 47 | ||
47 | // force content to be refreshed in case on bad fetch in the v1 installation | 48 | // force content to be refreshed in case on bad fetch in the v1 installation |
@@ -56,4 +57,14 @@ class WallabagV1Import extends WallabagImport | |||
56 | 57 | ||
57 | return $data; | 58 | return $data; |
58 | } | 59 | } |
60 | |||
61 | /** | ||
62 | * {@inheritdoc} | ||
63 | */ | ||
64 | protected function setEntryAsRead(array $importedEntry) | ||
65 | { | ||
66 | $importedEntry['is_read'] = 1; | ||
67 | |||
68 | return $importedEntry; | ||
69 | } | ||
59 | } | 70 | } |
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php index d0035b63..37c8ca14 100644 --- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php +++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php | |||
@@ -31,12 +31,22 @@ class WallabagV2Import extends WallabagImport | |||
31 | /** | 31 | /** |
32 | * {@inheritdoc} | 32 | * {@inheritdoc} |
33 | */ | 33 | */ |
34 | protected function prepareEntry($entry = [], $markAsRead = false) | 34 | protected function prepareEntry($entry = []) |
35 | { | 35 | { |
36 | return [ | 36 | return [ |
37 | 'html' => $entry['content'], | 37 | 'html' => $entry['content'], |
38 | 'content_type' => $entry['mimetype'], | 38 | 'content_type' => $entry['mimetype'], |
39 | 'is_archived' => ($entry['is_archived'] || $markAsRead), | 39 | 'is_archived' => ($entry['is_archived'] || $this->markAsRead), |
40 | ] + $entry; | 40 | ] + $entry; |
41 | } | 41 | } |
42 | |||
43 | /** | ||
44 | * {@inheritdoc} | ||
45 | */ | ||
46 | protected function setEntryAsRead(array $importedEntry) | ||
47 | { | ||
48 | $importedEntry['is_archived'] = 1; | ||
49 | |||
50 | return $importedEntry; | ||
51 | } | ||
42 | } | 52 | } |