diff options
author | Jeremy Benoist <j0k3r@users.noreply.github.com> | 2016-01-07 22:15:08 +0100 |
---|---|---|
committer | Jeremy Benoist <j0k3r@users.noreply.github.com> | 2016-01-07 22:15:08 +0100 |
commit | 39643c6b76d92d509b1af0228b6379d7fdce8a1c (patch) | |
tree | 931dceef7dbc8ae9911d01ded709d558417a6cdd /src/Wallabag/ImportBundle/Import | |
parent | 488a468e3e11ff0ab6284afe232bf0f7fa68a8eb (diff) | |
parent | b88cf91fc8371194df78e690983c61ea94f266cd (diff) | |
download | wallabag-39643c6b76d92d509b1af0228b6379d7fdce8a1c.tar.gz wallabag-39643c6b76d92d509b1af0228b6379d7fdce8a1c.tar.zst wallabag-39643c6b76d92d509b1af0228b6379d7fdce8a1c.zip |
Merge pull request #1493 from wallabag/v2-pocket-import2.0.0-alpha.1
v2 – 1st draft for Pocket import via API & Wallabag v1 import
Diffstat (limited to 'src/Wallabag/ImportBundle/Import')
-rw-r--r-- | src/Wallabag/ImportBundle/Import/ImportChain.php | 34 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/ImportCompilerPass.php | 33 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/ImportInterface.php | 45 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/PocketImport.php | 267 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/WallabagV1Import.php | 159 |
5 files changed, 538 insertions, 0 deletions
diff --git a/src/Wallabag/ImportBundle/Import/ImportChain.php b/src/Wallabag/ImportBundle/Import/ImportChain.php new file mode 100644 index 00000000..9dd77956 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ImportChain.php | |||
@@ -0,0 +1,34 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | class ImportChain | ||
6 | { | ||
7 | private $imports; | ||
8 | |||
9 | public function __construct() | ||
10 | { | ||
11 | $this->imports = []; | ||
12 | } | ||
13 | |||
14 | /** | ||
15 | * Add an import to the chain. | ||
16 | * | ||
17 | * @param ImportInterface $import | ||
18 | * @param string $alias | ||
19 | */ | ||
20 | public function addImport(ImportInterface $import, $alias) | ||
21 | { | ||
22 | $this->imports[$alias] = $import; | ||
23 | } | ||
24 | |||
25 | /** | ||
26 | * Get all imports. | ||
27 | * | ||
28 | * @return array<ImportInterface> | ||
29 | */ | ||
30 | public function getAll() | ||
31 | { | ||
32 | return $this->imports; | ||
33 | } | ||
34 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php b/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php new file mode 100644 index 00000000..a363a566 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ImportCompilerPass.php | |||
@@ -0,0 +1,33 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Symfony\Component\DependencyInjection\ContainerBuilder; | ||
6 | use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; | ||
7 | use Symfony\Component\DependencyInjection\Reference; | ||
8 | |||
9 | class ImportCompilerPass implements CompilerPassInterface | ||
10 | { | ||
11 | public function process(ContainerBuilder $container) | ||
12 | { | ||
13 | if (!$container->hasDefinition('wallabag_import.chain')) { | ||
14 | return; | ||
15 | } | ||
16 | |||
17 | $definition = $container->getDefinition( | ||
18 | 'wallabag_import.chain' | ||
19 | ); | ||
20 | |||
21 | $taggedServices = $container->findTaggedServiceIds( | ||
22 | 'wallabag_import.import' | ||
23 | ); | ||
24 | foreach ($taggedServices as $id => $tagAttributes) { | ||
25 | foreach ($tagAttributes as $attributes) { | ||
26 | $definition->addMethodCall( | ||
27 | 'addImport', | ||
28 | [new Reference($id), $attributes['alias']] | ||
29 | ); | ||
30 | } | ||
31 | } | ||
32 | } | ||
33 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/ImportInterface.php b/src/Wallabag/ImportBundle/Import/ImportInterface.php new file mode 100644 index 00000000..25dc0d85 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ImportInterface.php | |||
@@ -0,0 +1,45 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Psr\Log\LoggerAwareInterface; | ||
6 | |||
7 | interface ImportInterface extends LoggerAwareInterface | ||
8 | { | ||
9 | /** | ||
10 | * Name of the import. | ||
11 | * | ||
12 | * @return string | ||
13 | */ | ||
14 | public function getName(); | ||
15 | |||
16 | /** | ||
17 | * Url to start the import. | ||
18 | * | ||
19 | * @return string | ||
20 | */ | ||
21 | public function getUrl(); | ||
22 | |||
23 | /** | ||
24 | * Description of the import. | ||
25 | * | ||
26 | * @return string | ||
27 | */ | ||
28 | public function getDescription(); | ||
29 | |||
30 | /** | ||
31 | * Import content using the user token. | ||
32 | * | ||
33 | * @return bool | ||
34 | */ | ||
35 | public function import(); | ||
36 | |||
37 | /** | ||
38 | * Return an array with summary info about the import, with keys: | ||
39 | * - skipped | ||
40 | * - imported. | ||
41 | * | ||
42 | * @return array | ||
43 | */ | ||
44 | public function getSummary(); | ||
45 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php new file mode 100644 index 00000000..cdcec1e2 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php | |||
@@ -0,0 +1,267 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Psr\Log\LoggerInterface; | ||
6 | use Psr\Log\NullLogger; | ||
7 | use Doctrine\ORM\EntityManager; | ||
8 | use GuzzleHttp\Client; | ||
9 | use GuzzleHttp\Exception\RequestException; | ||
10 | use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; | ||
11 | use Wallabag\CoreBundle\Entity\Entry; | ||
12 | use Wallabag\CoreBundle\Entity\Tag; | ||
13 | use Wallabag\CoreBundle\Helper\ContentProxy; | ||
14 | |||
15 | class PocketImport implements ImportInterface | ||
16 | { | ||
17 | private $user; | ||
18 | private $em; | ||
19 | private $contentProxy; | ||
20 | private $logger; | ||
21 | private $client; | ||
22 | private $consumerKey; | ||
23 | private $skippedEntries = 0; | ||
24 | private $importedEntries = 0; | ||
25 | protected $accessToken; | ||
26 | private $translator; | ||
27 | |||
28 | public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, $consumerKey) | ||
29 | { | ||
30 | $this->user = $tokenStorage->getToken()->getUser(); | ||
31 | $this->em = $em; | ||
32 | $this->contentProxy = $contentProxy; | ||
33 | $this->consumerKey = $consumerKey; | ||
34 | $this->logger = new NullLogger(); | ||
35 | } | ||
36 | |||
37 | public function setLogger(LoggerInterface $logger) | ||
38 | { | ||
39 | $this->logger = $logger; | ||
40 | } | ||
41 | |||
42 | /** | ||
43 | * {@inheritdoc} | ||
44 | */ | ||
45 | public function getName() | ||
46 | { | ||
47 | return 'Pocket'; | ||
48 | } | ||
49 | |||
50 | /** | ||
51 | * {@inheritdoc} | ||
52 | */ | ||
53 | public function getUrl() | ||
54 | { | ||
55 | return 'import_pocket'; | ||
56 | } | ||
57 | |||
58 | /** | ||
59 | * {@inheritdoc} | ||
60 | */ | ||
61 | public function getDescription() | ||
62 | { | ||
63 | return 'This importer will import all your Pocket data. Pocket doesn\'t allow us to retrieve content from their service, so the readable content of each article will be re-fetched by wallabag.'; | ||
64 | } | ||
65 | |||
66 | /** | ||
67 | * Return the oauth url to authenticate the client. | ||
68 | * | ||
69 | * @param string $redirectUri Redirect url in case of error | ||
70 | * | ||
71 | * @return string request_token for callback method | ||
72 | */ | ||
73 | public function getRequestToken($redirectUri) | ||
74 | { | ||
75 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', | ||
76 | [ | ||
77 | 'body' => json_encode([ | ||
78 | 'consumer_key' => $this->consumerKey, | ||
79 | 'redirect_uri' => $redirectUri, | ||
80 | ]), | ||
81 | ] | ||
82 | ); | ||
83 | |||
84 | try { | ||
85 | $response = $this->client->send($request); | ||
86 | } catch (RequestException $e) { | ||
87 | $this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]); | ||
88 | |||
89 | return false; | ||
90 | } | ||
91 | |||
92 | return $response->json()['code']; | ||
93 | } | ||
94 | |||
95 | /** | ||
96 | * Usually called by the previous callback to authorize the client. | ||
97 | * Then it return a token that can be used for next requests. | ||
98 | * | ||
99 | * @param string $code request_token from getRequestToken | ||
100 | * | ||
101 | * @return bool | ||
102 | */ | ||
103 | public function authorize($code) | ||
104 | { | ||
105 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', | ||
106 | [ | ||
107 | 'body' => json_encode([ | ||
108 | 'consumer_key' => $this->consumerKey, | ||
109 | 'code' => $code, | ||
110 | ]), | ||
111 | ] | ||
112 | ); | ||
113 | |||
114 | try { | ||
115 | $response = $this->client->send($request); | ||
116 | } catch (RequestException $e) { | ||
117 | $this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]); | ||
118 | |||
119 | return false; | ||
120 | } | ||
121 | |||
122 | $this->accessToken = $response->json()['access_token']; | ||
123 | |||
124 | return true; | ||
125 | } | ||
126 | |||
127 | /** | ||
128 | * {@inheritdoc} | ||
129 | */ | ||
130 | public function import() | ||
131 | { | ||
132 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', | ||
133 | [ | ||
134 | 'body' => json_encode([ | ||
135 | 'consumer_key' => $this->consumerKey, | ||
136 | 'access_token' => $this->accessToken, | ||
137 | 'detailType' => 'complete', | ||
138 | 'state' => 'all', | ||
139 | 'sort' => 'oldest', | ||
140 | ]), | ||
141 | ] | ||
142 | ); | ||
143 | |||
144 | try { | ||
145 | $response = $this->client->send($request); | ||
146 | } catch (RequestException $e) { | ||
147 | $this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]); | ||
148 | |||
149 | return false; | ||
150 | } | ||
151 | |||
152 | $entries = $response->json(); | ||
153 | |||
154 | $this->parseEntries($entries['list']); | ||
155 | |||
156 | return true; | ||
157 | } | ||
158 | |||
159 | /** | ||
160 | * {@inheritdoc} | ||
161 | */ | ||
162 | public function getSummary() | ||
163 | { | ||
164 | return [ | ||
165 | 'skipped' => $this->skippedEntries, | ||
166 | 'imported' => $this->importedEntries, | ||
167 | ]; | ||
168 | } | ||
169 | |||
170 | /** | ||
171 | * Set the Guzzle client. | ||
172 | * | ||
173 | * @param Client $client | ||
174 | */ | ||
175 | public function setClient(Client $client) | ||
176 | { | ||
177 | $this->client = $client; | ||
178 | } | ||
179 | |||
180 | /** | ||
181 | * @todo move that in a more global place | ||
182 | */ | ||
183 | private function assignTagsToEntry(Entry $entry, $tags) | ||
184 | { | ||
185 | foreach ($tags as $tag) { | ||
186 | $label = trim($tag['tag']); | ||
187 | $tagEntity = $this->em | ||
188 | ->getRepository('WallabagCoreBundle:Tag') | ||
189 | ->findOneByLabel($label); | ||
190 | |||
191 | if (is_object($tagEntity)) { | ||
192 | $entry->addTag($tagEntity); | ||
193 | } else { | ||
194 | $newTag = new Tag(); | ||
195 | $newTag->setLabel($label); | ||
196 | |||
197 | $entry->addTag($newTag); | ||
198 | } | ||
199 | $this->em->flush(); | ||
200 | } | ||
201 | } | ||
202 | |||
203 | /** | ||
204 | * @see https://getpocket.com/developer/docs/v3/retrieve | ||
205 | * | ||
206 | * @param $entries | ||
207 | */ | ||
208 | private function parseEntries($entries) | ||
209 | { | ||
210 | $i = 1; | ||
211 | |||
212 | foreach ($entries as $pocketEntry) { | ||
213 | $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; | ||
214 | |||
215 | $existingEntry = $this->em | ||
216 | ->getRepository('WallabagCoreBundle:Entry') | ||
217 | ->existByUrlAndUserId($url, $this->user->getId()); | ||
218 | |||
219 | if (false !== $existingEntry) { | ||
220 | ++$this->skippedEntries; | ||
221 | continue; | ||
222 | } | ||
223 | |||
224 | $entry = new Entry($this->user); | ||
225 | $entry = $this->contentProxy->updateEntry($entry, $url); | ||
226 | |||
227 | // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted | ||
228 | if ($pocketEntry['status'] == 1) { | ||
229 | $entry->setArchived(true); | ||
230 | } | ||
231 | |||
232 | // 0 or 1 - 1 If the item is favorited | ||
233 | if ($pocketEntry['favorite'] == 1) { | ||
234 | $entry->setStarred(true); | ||
235 | } | ||
236 | |||
237 | $title = 'Untitled'; | ||
238 | if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { | ||
239 | $title = $pocketEntry['resolved_title']; | ||
240 | } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { | ||
241 | $title = $pocketEntry['given_title']; | ||
242 | } | ||
243 | |||
244 | $entry->setTitle($title); | ||
245 | |||
246 | // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image | ||
247 | if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { | ||
248 | $entry->setPreviewPicture($pocketEntry['images'][1]['src']); | ||
249 | } | ||
250 | |||
251 | if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { | ||
252 | $this->assignTagsToEntry($entry, $pocketEntry['tags']); | ||
253 | } | ||
254 | |||
255 | $this->em->persist($entry); | ||
256 | ++$this->importedEntries; | ||
257 | |||
258 | // flush every 20 entries | ||
259 | if (($i % 20) === 0) { | ||
260 | $this->em->flush(); | ||
261 | } | ||
262 | ++$i; | ||
263 | } | ||
264 | |||
265 | $this->em->flush(); | ||
266 | } | ||
267 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php new file mode 100644 index 00000000..393089d6 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php | |||
@@ -0,0 +1,159 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Psr\Log\LoggerInterface; | ||
6 | use Psr\Log\NullLogger; | ||
7 | use Doctrine\ORM\EntityManager; | ||
8 | use Wallabag\CoreBundle\Entity\Entry; | ||
9 | use Wallabag\UserBundle\Entity\User; | ||
10 | use Wallabag\CoreBundle\Tools\Utils; | ||
11 | |||
12 | class WallabagV1Import implements ImportInterface | ||
13 | { | ||
14 | private $user; | ||
15 | private $em; | ||
16 | private $logger; | ||
17 | private $skippedEntries = 0; | ||
18 | private $importedEntries = 0; | ||
19 | private $filepath; | ||
20 | |||
21 | public function __construct(EntityManager $em) | ||
22 | { | ||
23 | $this->em = $em; | ||
24 | $this->logger = new NullLogger(); | ||
25 | } | ||
26 | |||
27 | public function setLogger(LoggerInterface $logger) | ||
28 | { | ||
29 | $this->logger = $logger; | ||
30 | } | ||
31 | |||
32 | /** | ||
33 | * We define the user in a custom call because on the import command there is no logged in user. | ||
34 | * So we can't retrieve user from the `security.token_storage` service. | ||
35 | * | ||
36 | * @param User $user | ||
37 | */ | ||
38 | public function setUser(User $user) | ||
39 | { | ||
40 | $this->user = $user; | ||
41 | |||
42 | return $this; | ||
43 | } | ||
44 | |||
45 | /** | ||
46 | * {@inheritdoc} | ||
47 | */ | ||
48 | public function getName() | ||
49 | { | ||
50 | return 'wallabag v1'; | ||
51 | } | ||
52 | |||
53 | /** | ||
54 | * {@inheritdoc} | ||
55 | */ | ||
56 | public function getUrl() | ||
57 | { | ||
58 | return 'import_wallabag_v1'; | ||
59 | } | ||
60 | |||
61 | /** | ||
62 | * {@inheritdoc} | ||
63 | */ | ||
64 | public function getDescription() | ||
65 | { | ||
66 | return 'This importer will import all your wallabag v1 articles. On your config page, click on "JSON export" in the "Export your wallabag data" section. You will have a "wallabag-export-1-xxxx-xx-xx.json" file.'; | ||
67 | } | ||
68 | |||
69 | /** | ||
70 | * {@inheritdoc} | ||
71 | */ | ||
72 | public function import() | ||
73 | { | ||
74 | if (!$this->user) { | ||
75 | $this->logger->error('WallabagV1Import: user is not defined'); | ||
76 | |||
77 | return false; | ||
78 | } | ||
79 | |||
80 | if (!file_exists($this->filepath) || !is_readable($this->filepath)) { | ||
81 | $this->logger->error('WallabagV1Import: unable to read file', array('filepath' => $this->filepath)); | ||
82 | |||
83 | return false; | ||
84 | } | ||
85 | |||
86 | $data = json_decode(file_get_contents($this->filepath), true); | ||
87 | |||
88 | if (empty($data)) { | ||
89 | return false; | ||
90 | } | ||
91 | |||
92 | $this->parseEntries($data); | ||
93 | |||
94 | return true; | ||
95 | } | ||
96 | |||
97 | /** | ||
98 | * {@inheritdoc} | ||
99 | */ | ||
100 | public function getSummary() | ||
101 | { | ||
102 | return [ | ||
103 | 'skipped' => $this->skippedEntries, | ||
104 | 'imported' => $this->importedEntries, | ||
105 | ]; | ||
106 | } | ||
107 | |||
108 | /** | ||
109 | * Set file path to the json file. | ||
110 | * | ||
111 | * @param string $filepath | ||
112 | */ | ||
113 | public function setFilepath($filepath) | ||
114 | { | ||
115 | $this->filepath = $filepath; | ||
116 | |||
117 | return $this; | ||
118 | } | ||
119 | |||
120 | /** | ||
121 | * @param $entries | ||
122 | */ | ||
123 | private function parseEntries($entries) | ||
124 | { | ||
125 | $i = 1; | ||
126 | |||
127 | foreach ($entries as $importedEntry) { | ||
128 | $existingEntry = $this->em | ||
129 | ->getRepository('WallabagCoreBundle:Entry') | ||
130 | ->existByUrlAndUserId($importedEntry['url'], $this->user->getId()); | ||
131 | |||
132 | if (false !== $existingEntry) { | ||
133 | ++$this->skippedEntries; | ||
134 | continue; | ||
135 | } | ||
136 | |||
137 | // @see ContentProxy->updateEntry | ||
138 | $entry = new Entry($this->user); | ||
139 | $entry->setUrl($importedEntry['url']); | ||
140 | $entry->setTitle($importedEntry['title']); | ||
141 | $entry->setArchived($importedEntry['is_read']); | ||
142 | $entry->setStarred($importedEntry['is_fav']); | ||
143 | $entry->setContent($importedEntry['content']); | ||
144 | $entry->setReadingTime(Utils::getReadingTime($importedEntry['content'])); | ||
145 | $entry->setDomainName(parse_url($importedEntry['url'], PHP_URL_HOST)); | ||
146 | |||
147 | $this->em->persist($entry); | ||
148 | ++$this->importedEntries; | ||
149 | |||
150 | // flush every 20 entries | ||
151 | if (($i % 20) === 0) { | ||
152 | $this->em->flush(); | ||
153 | } | ||
154 | ++$i; | ||
155 | } | ||
156 | |||
157 | $this->em->flush(); | ||
158 | } | ||
159 | } | ||