diff options
Diffstat (limited to 'src/Wallabag/ImportBundle/Import/PocketImport.php')
-rw-r--r-- | src/Wallabag/ImportBundle/Import/PocketImport.php | 218 |
1 files changed, 103 insertions, 115 deletions
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index a6f905b1..e00eb44b 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php | |||
@@ -6,31 +6,34 @@ use Psr\Log\NullLogger; | |||
6 | use Doctrine\ORM\EntityManager; | 6 | use Doctrine\ORM\EntityManager; |
7 | use GuzzleHttp\Client; | 7 | use GuzzleHttp\Client; |
8 | use GuzzleHttp\Exception\RequestException; | 8 | use GuzzleHttp\Exception\RequestException; |
9 | use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; | ||
10 | use Wallabag\CoreBundle\Entity\Entry; | 9 | use Wallabag\CoreBundle\Entity\Entry; |
11 | use Wallabag\CoreBundle\Helper\ContentProxy; | 10 | use Wallabag\CoreBundle\Helper\ContentProxy; |
12 | use Craue\ConfigBundle\Util\Config; | ||
13 | 11 | ||
14 | class PocketImport extends AbstractImport | 12 | class PocketImport extends AbstractImport |
15 | { | 13 | { |
16 | private $user; | ||
17 | private $client; | 14 | private $client; |
18 | private $consumerKey; | 15 | private $accessToken; |
19 | private $skippedEntries = 0; | ||
20 | private $importedEntries = 0; | ||
21 | private $markAsRead; | ||
22 | protected $accessToken; | ||
23 | 16 | ||
24 | public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, Config $craueConfig) | 17 | const NB_ELEMENTS = 5000; |
18 | |||
19 | public function __construct(EntityManager $em, ContentProxy $contentProxy) | ||
25 | { | 20 | { |
26 | $this->user = $tokenStorage->getToken()->getUser(); | ||
27 | $this->em = $em; | 21 | $this->em = $em; |
28 | $this->contentProxy = $contentProxy; | 22 | $this->contentProxy = $contentProxy; |
29 | $this->consumerKey = $craueConfig->get('pocket_consumer_key'); | ||
30 | $this->logger = new NullLogger(); | 23 | $this->logger = new NullLogger(); |
31 | } | 24 | } |
32 | 25 | ||
33 | /** | 26 | /** |
27 | * Only used for test purpose. | ||
28 | * | ||
29 | * @return string | ||
30 | */ | ||
31 | public function getAccessToken() | ||
32 | { | ||
33 | return $this->accessToken; | ||
34 | } | ||
35 | |||
36 | /** | ||
34 | * {@inheritdoc} | 37 | * {@inheritdoc} |
35 | */ | 38 | */ |
36 | public function getName() | 39 | public function getName() |
@@ -66,7 +69,7 @@ class PocketImport extends AbstractImport | |||
66 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', | 69 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', |
67 | [ | 70 | [ |
68 | 'body' => json_encode([ | 71 | 'body' => json_encode([ |
69 | 'consumer_key' => $this->consumerKey, | 72 | 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), |
70 | 'redirect_uri' => $redirectUri, | 73 | 'redirect_uri' => $redirectUri, |
71 | ]), | 74 | ]), |
72 | ] | 75 | ] |
@@ -96,7 +99,7 @@ class PocketImport extends AbstractImport | |||
96 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', | 99 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', |
97 | [ | 100 | [ |
98 | 'body' => json_encode([ | 101 | 'body' => json_encode([ |
99 | 'consumer_key' => $this->consumerKey, | 102 | 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), |
100 | 'code' => $code, | 103 | 'code' => $code, |
101 | ]), | 104 | ]), |
102 | ] | 105 | ] |
@@ -116,38 +119,22 @@ class PocketImport extends AbstractImport | |||
116 | } | 119 | } |
117 | 120 | ||
118 | /** | 121 | /** |
119 | * Set whether articles must be all marked as read. | ||
120 | * | ||
121 | * @param bool $markAsRead | ||
122 | */ | ||
123 | public function setMarkAsRead($markAsRead) | ||
124 | { | ||
125 | $this->markAsRead = $markAsRead; | ||
126 | |||
127 | return $this; | ||
128 | } | ||
129 | |||
130 | /** | ||
131 | * Get whether articles must be all marked as read. | ||
132 | */ | ||
133 | public function getMarkAsRead() | ||
134 | { | ||
135 | return $this->markAsRead; | ||
136 | } | ||
137 | |||
138 | /** | ||
139 | * {@inheritdoc} | 122 | * {@inheritdoc} |
140 | */ | 123 | */ |
141 | public function import() | 124 | public function import($offset = 0) |
142 | { | 125 | { |
126 | static $run = 0; | ||
127 | |||
143 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', | 128 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', |
144 | [ | 129 | [ |
145 | 'body' => json_encode([ | 130 | 'body' => json_encode([ |
146 | 'consumer_key' => $this->consumerKey, | 131 | 'consumer_key' => $this->user->getConfig()->getPocketConsumerKey(), |
147 | 'access_token' => $this->accessToken, | 132 | 'access_token' => $this->accessToken, |
148 | 'detailType' => 'complete', | 133 | 'detailType' => 'complete', |
149 | 'state' => 'all', | 134 | 'state' => 'all', |
150 | 'sort' => 'oldest', | 135 | 'sort' => 'newest', |
136 | 'count' => self::NB_ELEMENTS, | ||
137 | 'offset' => $offset, | ||
151 | ]), | 138 | ]), |
152 | ] | 139 | ] |
153 | ); | 140 | ); |
@@ -162,20 +149,24 @@ class PocketImport extends AbstractImport | |||
162 | 149 | ||
163 | $entries = $response->json(); | 150 | $entries = $response->json(); |
164 | 151 | ||
165 | $this->parseEntries($entries['list']); | 152 | if ($this->producer) { |
153 | $this->parseEntriesForProducer($entries['list']); | ||
154 | } else { | ||
155 | $this->parseEntries($entries['list']); | ||
156 | } | ||
166 | 157 | ||
167 | return true; | 158 | // if we retrieve exactly the amount of items requested it means we can get more |
168 | } | 159 | // re-call import and offset item by the amount previous received: |
160 | // - first call get 5k offset 0 | ||
161 | // - second call get 5k offset 5k | ||
162 | // - and so on | ||
163 | if (count($entries['list']) === self::NB_ELEMENTS) { | ||
164 | ++$run; | ||
169 | 165 | ||
170 | /** | 166 | return $this->import(self::NB_ELEMENTS * $run); |
171 | * {@inheritdoc} | 167 | } |
172 | */ | 168 | |
173 | public function getSummary() | 169 | return true; |
174 | { | ||
175 | return [ | ||
176 | 'skipped' => $this->skippedEntries, | ||
177 | 'imported' => $this->importedEntries, | ||
178 | ]; | ||
179 | } | 170 | } |
180 | 171 | ||
181 | /** | 172 | /** |
@@ -189,77 +180,74 @@ class PocketImport extends AbstractImport | |||
189 | } | 180 | } |
190 | 181 | ||
191 | /** | 182 | /** |
192 | * @see https://getpocket.com/developer/docs/v3/retrieve | 183 | * {@inheritdoc} |
193 | * | 184 | * |
194 | * @param $entries | 185 | * @see https://getpocket.com/developer/docs/v3/retrieve |
195 | */ | 186 | */ |
196 | private function parseEntries($entries) | 187 | public function parseEntry(array $importedEntry) |
197 | { | 188 | { |
198 | $i = 1; | 189 | $url = isset($importedEntry['resolved_url']) && $importedEntry['resolved_url'] != '' ? $importedEntry['resolved_url'] : $importedEntry['given_url']; |
199 | 190 | ||
200 | foreach ($entries as $pocketEntry) { | 191 | $existingEntry = $this->em |
201 | $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; | 192 | ->getRepository('WallabagCoreBundle:Entry') |
202 | 193 | ->findByUrlAndUserId($url, $this->user->getId()); | |
203 | $existingEntry = $this->em | 194 | |
204 | ->getRepository('WallabagCoreBundle:Entry') | 195 | if (false !== $existingEntry) { |
205 | ->findByUrlAndUserId($url, $this->user->getId()); | 196 | ++$this->skippedEntries; |
206 | 197 | ||
207 | if (false !== $existingEntry) { | 198 | return; |
208 | ++$this->skippedEntries; | 199 | } |
209 | continue; | 200 | |
210 | } | 201 | $entry = new Entry($this->user); |
211 | 202 | $entry->setUrl($url); | |
212 | $entry = new Entry($this->user); | 203 | |
213 | $entry = $this->fetchContent($entry, $url); | 204 | // update entry with content (in case fetching failed, the given entry will be return) |
214 | 205 | $entry = $this->fetchContent($entry, $url); | |
215 | // jump to next entry in case of problem while getting content | 206 | |
216 | if (false === $entry) { | 207 | // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted |
217 | ++$this->skippedEntries; | 208 | $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead); |
218 | continue; | 209 | |
219 | } | 210 | // 0 or 1 - 1 If the item is starred |
220 | 211 | $entry->setStarred($importedEntry['favorite'] == 1); | |
221 | // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted | 212 | |
222 | if ($pocketEntry['status'] == 1 || $this->markAsRead) { | 213 | $title = 'Untitled'; |
223 | $entry->setArchived(true); | 214 | if (isset($importedEntry['resolved_title']) && $importedEntry['resolved_title'] != '') { |
224 | } | 215 | $title = $importedEntry['resolved_title']; |
225 | 216 | } elseif (isset($importedEntry['given_title']) && $importedEntry['given_title'] != '') { | |
226 | // 0 or 1 - 1 If the item is starred | 217 | $title = $importedEntry['given_title']; |
227 | if ($pocketEntry['favorite'] == 1) { | 218 | } |
228 | $entry->setStarred(true); | 219 | |
229 | } | 220 | $entry->setTitle($title); |
230 | 221 | ||
231 | $title = 'Untitled'; | 222 | // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image |
232 | if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { | 223 | if (isset($importedEntry['has_image']) && $importedEntry['has_image'] > 0 && isset($importedEntry['images'][1])) { |
233 | $title = $pocketEntry['resolved_title']; | 224 | $entry->setPreviewPicture($importedEntry['images'][1]['src']); |
234 | } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { | 225 | } |
235 | $title = $pocketEntry['given_title']; | 226 | |
236 | } | 227 | if (isset($importedEntry['tags']) && !empty($importedEntry['tags'])) { |
237 | 228 | $this->contentProxy->assignTagsToEntry( | |
238 | $entry->setTitle($title); | 229 | $entry, |
239 | 230 | array_keys($importedEntry['tags']) | |
240 | // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image | 231 | ); |
241 | if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { | ||
242 | $entry->setPreviewPicture($pocketEntry['images'][1]['src']); | ||
243 | } | ||
244 | |||
245 | if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { | ||
246 | $this->contentProxy->assignTagsToEntry( | ||
247 | $entry, | ||
248 | array_keys($pocketEntry['tags']) | ||
249 | ); | ||
250 | } | ||
251 | |||
252 | $this->em->persist($entry); | ||
253 | ++$this->importedEntries; | ||
254 | |||
255 | // flush every 20 entries | ||
256 | if (($i % 20) === 0) { | ||
257 | $this->em->flush(); | ||
258 | } | ||
259 | ++$i; | ||
260 | } | 232 | } |
261 | 233 | ||
262 | $this->em->flush(); | 234 | if (!empty($importedEntry['time_added'])) { |
263 | $this->em->clear(); | 235 | $entry->setCreatedAt((new \DateTime())->setTimestamp($importedEntry['time_added'])); |
236 | } | ||
237 | |||
238 | $this->em->persist($entry); | ||
239 | ++$this->importedEntries; | ||
240 | |||
241 | return $entry; | ||
242 | } | ||
243 | |||
244 | /** | ||
245 | * {@inheritdoc} | ||
246 | */ | ||
247 | protected function setEntryAsRead(array $importedEntry) | ||
248 | { | ||
249 | $importedEntry['status'] = '1'; | ||
250 | |||
251 | return $importedEntry; | ||
264 | } | 252 | } |
265 | } | 253 | } |