diff options
author | Jeremy Benoist <jeremy.benoist@gmail.com> | 2015-12-30 12:23:51 +0100 |
---|---|---|
committer | Jeremy Benoist <jeremy.benoist@gmail.com> | 2016-01-02 23:27:41 +0100 |
commit | 252ebd60719d32ec954d0519c9edf2b52b03310c (patch) | |
tree | 044c97abeda75c33901d8bfcd33fa107279b1778 | |
parent | b4b592a0c0ee356e81775baf8f9976288d7b686c (diff) | |
download | wallabag-252ebd60719d32ec954d0519c9edf2b52b03310c.tar.gz wallabag-252ebd60719d32ec954d0519c9edf2b52b03310c.tar.zst wallabag-252ebd60719d32ec954d0519c9edf2b52b03310c.zip |
Rewrote Pocket Import
For the moment, we won't do a queue system, just a plain synchronous import.
We also use ContentProxy to grab content for each article from Pocket.
Error from Pocket are now logged using the logger.
The ImportInterface need to be simple and not related to oAuth (not all import will use that method).
9 files changed, 357 insertions, 210 deletions
diff --git a/src/Wallabag/CoreBundle/Tools/Utils.php b/src/Wallabag/CoreBundle/Tools/Utils.php index b501ce65..a16baca9 100644 --- a/src/Wallabag/CoreBundle/Tools/Utils.php +++ b/src/Wallabag/CoreBundle/Tools/Utils.php | |||
@@ -27,16 +27,6 @@ class Utils | |||
27 | } | 27 | } |
28 | 28 | ||
29 | /** | 29 | /** |
30 | * @param $words | ||
31 | * | ||
32 | * @return float | ||
33 | */ | ||
34 | public static function convertWordsToMinutes($words) | ||
35 | { | ||
36 | return floor($words / 200); | ||
37 | } | ||
38 | |||
39 | /** | ||
40 | * For a given text, we calculate reading time for an article | 30 | * For a given text, we calculate reading time for an article |
41 | * based on 200 words per minute. | 31 | * based on 200 words per minute. |
42 | * | 32 | * |
@@ -46,6 +36,6 @@ class Utils | |||
46 | */ | 36 | */ |
47 | public static function getReadingTime($text) | 37 | public static function getReadingTime($text) |
48 | { | 38 | { |
49 | return self::convertWordsToMinutes(str_word_count(strip_tags($text))); | 39 | return floor(str_word_count(strip_tags($text)) / 200); |
50 | } | 40 | } |
51 | } | 41 | } |
diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 6ebd6a0a..2a0d6ab5 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php | |||
@@ -4,58 +4,14 @@ namespace Wallabag\ImportBundle\Controller; | |||
4 | 4 | ||
5 | use Symfony\Bundle\FrameworkBundle\Controller\Controller; | 5 | use Symfony\Bundle\FrameworkBundle\Controller\Controller; |
6 | use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route; | 6 | use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route; |
7 | use Symfony\Component\Console\Input\ArrayInput; | ||
8 | use Symfony\Component\Console\Output\NullOutput; | ||
9 | use Symfony\Component\HttpFoundation\Request; | ||
10 | use Wallabag\ImportBundle\Command\ImportCommand; | ||
11 | use Wallabag\ImportBundle\Form\Type\UploadImportType; | ||
12 | 7 | ||
13 | class ImportController extends Controller | 8 | class ImportController extends Controller |
14 | { | 9 | { |
15 | /** | 10 | /** |
16 | * @Route("/import", name="import") | 11 | * @Route("/import", name="import") |
17 | */ | 12 | */ |
18 | public function importAction(Request $request) | 13 | public function importAction() |
19 | { | 14 | { |
20 | $importForm = $this->createForm(new UploadImportType()); | 15 | return $this->render('WallabagImportBundle:Import:index.html.twig', []); |
21 | $importForm->handleRequest($request); | ||
22 | $user = $this->getUser(); | ||
23 | |||
24 | if ($importForm->isValid()) { | ||
25 | $file = $importForm->get('file')->getData(); | ||
26 | $name = $user->getId().'.json'; | ||
27 | $dir = __DIR__.'/../../../../web/uploads/import'; | ||
28 | |||
29 | if (in_array($file->getMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($dir, $name)) { | ||
30 | $command = new ImportCommand(); | ||
31 | $command->setContainer($this->container); | ||
32 | $input = new ArrayInput(array('userId' => $user->getId())); | ||
33 | $return = $command->run($input, new NullOutput()); | ||
34 | |||
35 | if ($return == 0) { | ||
36 | $this->get('session')->getFlashBag()->add( | ||
37 | 'notice', | ||
38 | 'Import successful' | ||
39 | ); | ||
40 | } else { | ||
41 | $this->get('session')->getFlashBag()->add( | ||
42 | 'notice', | ||
43 | 'Import failed' | ||
44 | ); | ||
45 | } | ||
46 | |||
47 | return $this->redirect('/'); | ||
48 | } else { | ||
49 | $this->get('session')->getFlashBag()->add( | ||
50 | 'notice', | ||
51 | 'Error while processing import. Please verify your import file.' | ||
52 | ); | ||
53 | } | ||
54 | } | ||
55 | |||
56 | return $this->render('WallabagImportBundle:Import:index.html.twig', array( | ||
57 | 'form' => array( | ||
58 | 'import' => $importForm->createView(), ), | ||
59 | )); | ||
60 | } | 16 | } |
61 | } | 17 | } |
diff --git a/src/Wallabag/ImportBundle/Controller/PocketController.php b/src/Wallabag/ImportBundle/Controller/PocketController.php index 2ab062e7..61eeba43 100644 --- a/src/Wallabag/ImportBundle/Controller/PocketController.php +++ b/src/Wallabag/ImportBundle/Controller/PocketController.php | |||
@@ -8,35 +8,56 @@ use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route; | |||
8 | class PocketController extends Controller | 8 | class PocketController extends Controller |
9 | { | 9 | { |
10 | /** | 10 | /** |
11 | * @Route("/import/pocket", name="pocket_import") | 11 | * @Route("/import/pocket", name="import_pocket") |
12 | */ | 12 | */ |
13 | public function indexAction() | 13 | public function indexAction() |
14 | { | 14 | { |
15 | return $this->render('WallabagImportBundle:Pocket:index.html.twig', array()); | 15 | return $this->render('WallabagImportBundle:Pocket:index.html.twig', []); |
16 | } | 16 | } |
17 | 17 | ||
18 | /** | 18 | /** |
19 | * @Route("/import/pocket/auth", name="pocket_auth") | 19 | * @Route("/import/pocket/auth", name="import_pocket_auth") |
20 | */ | 20 | */ |
21 | public function authAction() | 21 | public function authAction() |
22 | { | 22 | { |
23 | $pocket = $this->get('wallabag_import.pocket.import'); | 23 | $requestToken = $this->get('wallabag_import.pocket.import') |
24 | $authUrl = $pocket->oAuthRequest( | 24 | ->getRequestToken($this->generateUrl('import', [], true)); |
25 | $this->generateUrl('import', array(), true), | 25 | |
26 | $this->generateUrl('pocket_callback', array(), true) | 26 | $this->get('session')->set('import.pocket.code', $requestToken); |
27 | ); | ||
28 | 27 | ||
29 | return $this->redirect($authUrl, 301); | 28 | return $this->redirect( |
29 | 'https://getpocket.com/auth/authorize?request_token='.$requestToken.'&redirect_uri='.$this->generateUrl('import_pocket_callback', [], true), | ||
30 | 301 | ||
31 | ); | ||
30 | } | 32 | } |
31 | 33 | ||
32 | /** | 34 | /** |
33 | * @Route("/import/pocket/callback", name="pocket_callback") | 35 | * @Route("/import/pocket/callback", name="import_pocket_callback") |
34 | */ | 36 | */ |
35 | public function callbackAction() | 37 | public function callbackAction() |
36 | { | 38 | { |
39 | $message = 'Import failed, please try again.'; | ||
37 | $pocket = $this->get('wallabag_import.pocket.import'); | 40 | $pocket = $this->get('wallabag_import.pocket.import'); |
38 | $accessToken = $pocket->oAuthAuthorize(); | 41 | |
39 | $pocket->import($accessToken); | 42 | // something bad happend on pocket side |
43 | if (false === $pocket->authorize($this->get('session')->get('import.pocket.code'))) { | ||
44 | $this->get('session')->getFlashBag()->add( | ||
45 | 'notice', | ||
46 | $message | ||
47 | ); | ||
48 | |||
49 | return $this->redirect($this->generateUrl('import_pocket')); | ||
50 | } | ||
51 | |||
52 | if (true === $pocket->import()) { | ||
53 | $summary = $pocket->getSummary(); | ||
54 | $message = $summary['imported'].' entrie(s) imported, '.$summary['skipped'].' already saved.'; | ||
55 | } | ||
56 | |||
57 | $this->get('session')->getFlashBag()->add( | ||
58 | 'notice', | ||
59 | $message | ||
60 | ); | ||
40 | 61 | ||
41 | return $this->redirect($this->generateUrl('homepage')); | 62 | return $this->redirect($this->generateUrl('homepage')); |
42 | } | 63 | } |
diff --git a/src/Wallabag/ImportBundle/Import/ImportInterface.php b/src/Wallabag/ImportBundle/Import/ImportInterface.php index 0f9b3256..8cf238aa 100644 --- a/src/Wallabag/ImportBundle/Import/ImportInterface.php +++ b/src/Wallabag/ImportBundle/Import/ImportInterface.php | |||
@@ -2,7 +2,9 @@ | |||
2 | 2 | ||
3 | namespace Wallabag\ImportBundle\Import; | 3 | namespace Wallabag\ImportBundle\Import; |
4 | 4 | ||
5 | interface ImportInterface | 5 | use Psr\Log\LoggerAwareInterface; |
6 | |||
7 | interface ImportInterface extends LoggerAwareInterface | ||
6 | { | 8 | { |
7 | /** | 9 | /** |
8 | * Name of the import. | 10 | * Name of the import. |
@@ -19,27 +21,18 @@ interface ImportInterface | |||
19 | public function getDescription(); | 21 | public function getDescription(); |
20 | 22 | ||
21 | /** | 23 | /** |
22 | * Return the oauth url to authenticate the client. | 24 | * Import content using the user token. |
23 | * | ||
24 | * @param string $redirectUri Redirect url in case of error | ||
25 | * @param string $callbackUri Url when the authentication is complete | ||
26 | * | ||
27 | * @return string | ||
28 | */ | ||
29 | public function oAuthRequest($redirectUri, $callbackUri); | ||
30 | |||
31 | /** | ||
32 | * Usually called by the previous callback to authorize the client. | ||
33 | * Then it return a token that can be used for next requests. | ||
34 | * | 25 | * |
35 | * @return string | 26 | * @return bool |
36 | */ | 27 | */ |
37 | public function oAuthAuthorize(); | 28 | public function import(); |
38 | 29 | ||
39 | /** | 30 | /** |
40 | * Import content using the user token. | 31 | * Return an array with summary info about the import, with keys: |
32 | * - skipped | ||
33 | * - imported. | ||
41 | * | 34 | * |
42 | * @param string $accessToken User access token | 35 | * @return array |
43 | */ | 36 | */ |
44 | public function import($accessToken); | 37 | public function getSummary(); |
45 | } | 38 | } |
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index e5c86f07..1710d9d3 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php | |||
@@ -2,29 +2,39 @@ | |||
2 | 2 | ||
3 | namespace Wallabag\ImportBundle\Import; | 3 | namespace Wallabag\ImportBundle\Import; |
4 | 4 | ||
5 | use Psr\Log\LoggerInterface; | ||
6 | use Psr\Log\NullLogger; | ||
5 | use Doctrine\ORM\EntityManager; | 7 | use Doctrine\ORM\EntityManager; |
6 | use GuzzleHttp\Client; | 8 | use GuzzleHttp\Client; |
7 | use Symfony\Component\HttpFoundation\Session\Session; | 9 | use GuzzleHttp\Exception\RequestException; |
8 | use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; | 10 | use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInterface; |
9 | use Wallabag\CoreBundle\Entity\Entry; | 11 | use Wallabag\CoreBundle\Entity\Entry; |
10 | use Wallabag\CoreBundle\Entity\Tag; | 12 | use Wallabag\CoreBundle\Entity\Tag; |
11 | use Wallabag\CoreBundle\Tools\Utils; | 13 | use Wallabag\CoreBundle\Helper\ContentProxy; |
12 | 14 | ||
13 | class PocketImport implements ImportInterface | 15 | class PocketImport implements ImportInterface |
14 | { | 16 | { |
15 | private $user; | 17 | private $user; |
16 | private $session; | ||
17 | private $em; | 18 | private $em; |
19 | private $contentProxy; | ||
20 | private $logger; | ||
18 | private $consumerKey; | 21 | private $consumerKey; |
19 | private $skippedEntries = 0; | 22 | private $skippedEntries = 0; |
20 | private $importedEntries = 0; | 23 | private $importedEntries = 0; |
24 | protected $accessToken; | ||
21 | 25 | ||
22 | public function __construct(TokenStorageInterface $tokenStorage, Session $session, EntityManager $em, $consumerKey) | 26 | public function __construct(TokenStorageInterface $tokenStorage, EntityManager $em, ContentProxy $contentProxy, $consumerKey) |
23 | { | 27 | { |
24 | $this->user = $tokenStorage->getToken()->getUser(); | 28 | $this->user = $tokenStorage->getToken()->getUser(); |
25 | $this->session = $session; | ||
26 | $this->em = $em; | 29 | $this->em = $em; |
30 | $this->contentProxy = $contentProxy; | ||
27 | $this->consumerKey = $consumerKey; | 31 | $this->consumerKey = $consumerKey; |
32 | $this->logger = new NullLogger(); | ||
33 | } | ||
34 | |||
35 | public function setLogger(LoggerInterface $logger) | ||
36 | { | ||
37 | $this->logger = $logger; | ||
28 | } | 38 | } |
29 | 39 | ||
30 | /** | 40 | /** |
@@ -44,9 +54,13 @@ class PocketImport implements ImportInterface | |||
44 | } | 54 | } |
45 | 55 | ||
46 | /** | 56 | /** |
47 | * {@inheritdoc} | 57 | * Return the oauth url to authenticate the client. |
58 | * | ||
59 | * @param string $redirectUri Redirect url in case of error | ||
60 | * | ||
61 | * @return string request_token for callback method | ||
48 | */ | 62 | */ |
49 | public function oAuthRequest($redirectUri, $callbackUri) | 63 | public function getRequestToken($redirectUri) |
50 | { | 64 | { |
51 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', | 65 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/request', |
52 | [ | 66 | [ |
@@ -57,44 +71,59 @@ class PocketImport implements ImportInterface | |||
57 | ] | 71 | ] |
58 | ); | 72 | ); |
59 | 73 | ||
60 | $response = $this->client->send($request); | 74 | try { |
61 | $values = $response->json(); | 75 | $response = $this->client->send($request); |
76 | } catch (RequestException $e) { | ||
77 | $this->logger->error(sprintf('PocketImport: Failed to request token: %s', $e->getMessage()), ['exception' => $e]); | ||
62 | 78 | ||
63 | // store code in session for callback method | 79 | return false; |
64 | $this->session->set('pocketCode', $values['code']); | 80 | } |
65 | 81 | ||
66 | return 'https://getpocket.com/auth/authorize?request_token='.$values['code'].'&redirect_uri='.$callbackUri; | 82 | return $response->json()['code']; |
67 | } | 83 | } |
68 | 84 | ||
69 | /** | 85 | /** |
70 | * {@inheritdoc} | 86 | * Usually called by the previous callback to authorize the client. |
87 | * Then it return a token that can be used for next requests. | ||
88 | * | ||
89 | * @param string $code request_token from getRequestToken | ||
90 | * | ||
91 | * @return bool | ||
71 | */ | 92 | */ |
72 | public function oAuthAuthorize() | 93 | public function authorize($code) |
73 | { | 94 | { |
74 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', | 95 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/oauth/authorize', |
75 | [ | 96 | [ |
76 | 'body' => json_encode([ | 97 | 'body' => json_encode([ |
77 | 'consumer_key' => $this->consumerKey, | 98 | 'consumer_key' => $this->consumerKey, |
78 | 'code' => $this->session->get('pocketCode'), | 99 | 'code' => $code, |
79 | ]), | 100 | ]), |
80 | ] | 101 | ] |
81 | ); | 102 | ); |
82 | 103 | ||
83 | $response = $this->client->send($request); | 104 | try { |
105 | $response = $this->client->send($request); | ||
106 | } catch (RequestException $e) { | ||
107 | $this->logger->error(sprintf('PocketImport: Failed to authorize client: %s', $e->getMessage()), ['exception' => $e]); | ||
84 | 108 | ||
85 | return $response->json()['access_token']; | 109 | return false; |
110 | } | ||
111 | |||
112 | $this->accessToken = $response->json()['access_token']; | ||
113 | |||
114 | return true; | ||
86 | } | 115 | } |
87 | 116 | ||
88 | /** | 117 | /** |
89 | * {@inheritdoc} | 118 | * {@inheritdoc} |
90 | */ | 119 | */ |
91 | public function import($accessToken) | 120 | public function import() |
92 | { | 121 | { |
93 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', | 122 | $request = $this->client->createRequest('POST', 'https://getpocket.com/v3/get', |
94 | [ | 123 | [ |
95 | 'body' => json_encode([ | 124 | 'body' => json_encode([ |
96 | 'consumer_key' => $this->consumerKey, | 125 | 'consumer_key' => $this->consumerKey, |
97 | 'access_token' => $accessToken, | 126 | 'access_token' => $this->accessToken, |
98 | 'detailType' => 'complete', | 127 | 'detailType' => 'complete', |
99 | 'state' => 'all', | 128 | 'state' => 'all', |
100 | 'sort' => 'oldest', | 129 | 'sort' => 'oldest', |
@@ -102,61 +131,45 @@ class PocketImport implements ImportInterface | |||
102 | ] | 131 | ] |
103 | ); | 132 | ); |
104 | 133 | ||
105 | $response = $this->client->send($request); | 134 | try { |
135 | $response = $this->client->send($request); | ||
136 | } catch (RequestException $e) { | ||
137 | $this->logger->error(sprintf('PocketImport: Failed to import: %s', $e->getMessage()), ['exception' => $e]); | ||
138 | |||
139 | return false; | ||
140 | } | ||
141 | |||
106 | $entries = $response->json(); | 142 | $entries = $response->json(); |
107 | 143 | ||
108 | $this->parsePocketEntries($entries['list']); | 144 | $this->parsePocketEntries($entries['list']); |
109 | 145 | ||
110 | $this->session->getFlashBag()->add( | 146 | return true; |
111 | 'notice', | ||
112 | $this->importedEntries.' entries imported, '.$this->skippedEntries.' already saved.' | ||
113 | ); | ||
114 | } | 147 | } |
115 | 148 | ||
116 | /** | 149 | /** |
117 | * Set the Guzzle client. | 150 | * {@inheritdoc} |
118 | * | ||
119 | * @param Client $client | ||
120 | */ | 151 | */ |
121 | public function setClient(Client $client) | 152 | public function getSummary() |
122 | { | 153 | { |
123 | $this->client = $client; | 154 | return [ |
155 | 'skipped' => $this->skippedEntries, | ||
156 | 'imported' => $this->importedEntries, | ||
157 | ]; | ||
124 | } | 158 | } |
125 | 159 | ||
126 | /** | 160 | /** |
127 | * Returns the good title for current entry. | 161 | * Set the Guzzle client. |
128 | * | ||
129 | * @param $pocketEntry | ||
130 | * | 162 | * |
131 | * @return string | 163 | * @param Client $client |
132 | */ | 164 | */ |
133 | private function guessTitle($pocketEntry) | 165 | public function setClient(Client $client) |
134 | { | 166 | { |
135 | if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { | 167 | $this->client = $client; |
136 | return $pocketEntry['resolved_title']; | ||
137 | } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { | ||
138 | return $pocketEntry['given_title']; | ||
139 | } | ||
140 | |||
141 | return 'Untitled'; | ||
142 | } | 168 | } |
143 | 169 | ||
144 | /** | 170 | /** |
145 | * Returns the good URL for current entry. | 171 | * @todo move that in a more global place |
146 | * | ||
147 | * @param $pocketEntry | ||
148 | * | ||
149 | * @return string | ||
150 | */ | 172 | */ |
151 | private function guessURL($pocketEntry) | ||
152 | { | ||
153 | if (isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '') { | ||
154 | return $pocketEntry['resolved_url']; | ||
155 | } | ||
156 | |||
157 | return $pocketEntry['given_url']; | ||
158 | } | ||
159 | |||
160 | private function assignTagsToEntry(Entry $entry, $tags) | 173 | private function assignTagsToEntry(Entry $entry, $tags) |
161 | { | 174 | { |
162 | foreach ($tags as $tag) { | 175 | foreach ($tags as $tag) { |
@@ -177,13 +190,16 @@ class PocketImport implements ImportInterface | |||
177 | } | 190 | } |
178 | 191 | ||
179 | /** | 192 | /** |
193 | * @see https://getpocket.com/developer/docs/v3/retrieve | ||
194 | * | ||
180 | * @param $entries | 195 | * @param $entries |
181 | */ | 196 | */ |
182 | private function parsePocketEntries($entries) | 197 | private function parsePocketEntries($entries) |
183 | { | 198 | { |
184 | foreach ($entries as $pocketEntry) { | 199 | foreach ($entries as $pocketEntry) { |
185 | $entry = new Entry($this->user); | 200 | $entry = new Entry($this->user); |
186 | $url = $this->guessURL($pocketEntry); | 201 | |
202 | $url = isset($pocketEntry['resolved_url']) && $pocketEntry['resolved_url'] != '' ? $pocketEntry['resolved_url'] : $pocketEntry['given_url']; | ||
187 | 203 | ||
188 | $existingEntry = $this->em | 204 | $existingEntry = $this->em |
189 | ->getRepository('WallabagCoreBundle:Entry') | 205 | ->getRepository('WallabagCoreBundle:Entry') |
@@ -194,31 +210,33 @@ class PocketImport implements ImportInterface | |||
194 | continue; | 210 | continue; |
195 | } | 211 | } |
196 | 212 | ||
197 | $entry->setUrl($url); | 213 | $entry = $this->contentProxy->updateEntry($entry, $url); |
198 | $entry->setDomainName(parse_url($url, PHP_URL_HOST)); | ||
199 | 214 | ||
215 | // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted | ||
200 | if ($pocketEntry['status'] == 1) { | 216 | if ($pocketEntry['status'] == 1) { |
201 | $entry->setArchived(true); | 217 | $entry->setArchived(true); |
202 | } | 218 | } |
219 | |||
220 | // 0 or 1 - 1 If the item is favorited | ||
203 | if ($pocketEntry['favorite'] == 1) { | 221 | if ($pocketEntry['favorite'] == 1) { |
204 | $entry->setStarred(true); | 222 | $entry->setStarred(true); |
205 | } | 223 | } |
206 | 224 | ||
207 | $entry->setTitle($this->guessTitle($pocketEntry)); | 225 | $title = 'Untitled'; |
208 | 226 | if (isset($pocketEntry['resolved_title']) && $pocketEntry['resolved_title'] != '') { | |
209 | if (isset($pocketEntry['excerpt'])) { | 227 | $title = $pocketEntry['resolved_title']; |
210 | $entry->setContent($pocketEntry['excerpt']); | 228 | } elseif (isset($pocketEntry['given_title']) && $pocketEntry['given_title'] != '') { |
229 | $title = $pocketEntry['given_title']; | ||
211 | } | 230 | } |
212 | 231 | ||
213 | if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0) { | 232 | $entry->setTitle($title); |
214 | $entry->setPreviewPicture($pocketEntry['image']['src']); | ||
215 | } | ||
216 | 233 | ||
217 | if (isset($pocketEntry['word_count'])) { | 234 | // 0, 1, or 2 - 1 if the item has images in it - 2 if the item is an image |
218 | $entry->setReadingTime(Utils::convertWordsToMinutes($pocketEntry['word_count'])); | 235 | if (isset($pocketEntry['has_image']) && $pocketEntry['has_image'] > 0 && isset($pocketEntry['images'][1])) { |
236 | $entry->setPreviewPicture($pocketEntry['images'][1]['src']); | ||
219 | } | 237 | } |
220 | 238 | ||
221 | if (!empty($pocketEntry['tags'])) { | 239 | if (isset($pocketEntry['tags']) && !empty($pocketEntry['tags'])) { |
222 | $this->assignTagsToEntry($entry, $pocketEntry['tags']); | 240 | $this->assignTagsToEntry($entry, $pocketEntry['tags']); |
223 | } | 241 | } |
224 | 242 | ||
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index ab516ca5..f421821c 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml | |||
@@ -1,14 +1,4 @@ | |||
1 | services: | 1 | services: |
2 | wallabag_import.pocket.import: | ||
3 | class: Wallabag\ImportBundle\Import\PocketImport | ||
4 | arguments: | ||
5 | - "@security.token_storage" | ||
6 | - "@session" | ||
7 | - "@doctrine.orm.entity_manager" | ||
8 | - %pocket_consumer_key% | ||
9 | calls: | ||
10 | - [ setClient, [ "@wallabag_import.pocket.client" ] ] | ||
11 | |||
12 | wallabag_import.pocket.client: | 2 | wallabag_import.pocket.client: |
13 | class: GuzzleHttp\Client | 3 | class: GuzzleHttp\Client |
14 | arguments: | 4 | arguments: |
@@ -17,3 +7,14 @@ services: | |||
17 | headers: | 7 | headers: |
18 | content-type: "application/json" | 8 | content-type: "application/json" |
19 | X-Accept: "application/json" | 9 | X-Accept: "application/json" |
10 | |||
11 | wallabag_import.pocket.import: | ||
12 | class: Wallabag\ImportBundle\Import\PocketImport | ||
13 | arguments: | ||
14 | - "@security.token_storage" | ||
15 | - "@doctrine.orm.entity_manager" | ||
16 | - "@wallabag_core.content_proxy" | ||
17 | - %pocket_consumer_key% | ||
18 | calls: | ||
19 | - [ setClient, [ "@wallabag_import.pocket.client" ] ] | ||
20 | - [ setLogger, [ "@logger" ]] | ||
diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index ee759a52..b068283a 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig | |||
@@ -8,35 +8,9 @@ | |||
8 | <div class="card-panel settings"> | 8 | <div class="card-panel settings"> |
9 | {% trans %}Welcome on wallabag importer. Please select your previous service that you want to migrate.{% endtrans %} | 9 | {% trans %}Welcome on wallabag importer. Please select your previous service that you want to migrate.{% endtrans %} |
10 | <ul> | 10 | <ul> |
11 | <li><a href="{{ path('pocket_import') }}">Pocket</a></li> | 11 | <li><a href="{{ path('import_pocket') }}">Pocket</a></li> |
12 | </ul> | 12 | </ul> |
13 | </div> | 13 | </div> |
14 | </div> | 14 | </div> |
15 | </div> | 15 | </div> |
16 | |||
17 | |||
18 | <div class="row"> | ||
19 | <div class="col s12"> | ||
20 | <div class="card-panel settings"> | ||
21 | <div class="row"> | ||
22 | <div class="col s12"> | ||
23 | <form action="{{ path('import') }}" method="post" {{ form_enctype(form.import) }}> | ||
24 | {{ form_errors(form.import) }} | ||
25 | <div class="row"> | ||
26 | <div class="input-field col s12"> | ||
27 | <p>{% trans %}Please select your wallabag export and click on the below button to upload and import it.{% endtrans %}</p> | ||
28 | {{ form_errors(form.import.file) }} | ||
29 | {{ form_widget(form.import.file) }} | ||
30 | </div> | ||
31 | </div> | ||
32 | <div class="hidden">{{ form_rest(form.import) }}</div> | ||
33 | <button class="btn waves-effect waves-light" type="submit" name="action"> | ||
34 | {% trans %}Upload file{% endtrans %} | ||
35 | </button> | ||
36 | </form> | ||
37 | </div> | ||
38 | </div> | ||
39 | </div> | ||
40 | </div> | ||
41 | </div> | ||
42 | {% endblock %} | 16 | {% endblock %} |
diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig index df64e472..940fe4cc 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig | |||
@@ -7,7 +7,7 @@ | |||
7 | <div class="col s12"> | 7 | <div class="col s12"> |
8 | <div class="card-panel settings"> | 8 | <div class="card-panel settings"> |
9 | {% trans %}You can import your data from your Pocket account. You just have to click on the below button and authorize the application to connect to getpocket.com.{% endtrans %} | 9 | {% trans %}You can import your data from your Pocket account. You just have to click on the below button and authorize the application to connect to getpocket.com.{% endtrans %} |
10 | <form method="post" action="{{ path('pocket_auth') }}"> | 10 | <form method="post" action="{{ path('import_pocket_auth') }}"> |
11 | <input type="submit" value="Connect to Pocket and import data" /> | 11 | <input type="submit" value="Connect to Pocket and import data" /> |
12 | </form> | 12 | </form> |
13 | </div> | 13 | </div> |
diff --git a/src/Wallabag/ImportBundle/Tests/Import/PocketImportTest.php b/src/Wallabag/ImportBundle/Tests/Import/PocketImportTest.php index 4c718fa3..cf706fa9 100644 --- a/src/Wallabag/ImportBundle/Tests/Import/PocketImportTest.php +++ b/src/Wallabag/ImportBundle/Tests/Import/PocketImportTest.php | |||
@@ -4,19 +4,28 @@ namespace Wallabag\ImportBundle\Tests\Import; | |||
4 | 4 | ||
5 | use Wallabag\UserBundle\Entity\User; | 5 | use Wallabag\UserBundle\Entity\User; |
6 | use Wallabag\ImportBundle\Import\PocketImport; | 6 | use Wallabag\ImportBundle\Import\PocketImport; |
7 | use Symfony\Component\HttpFoundation\Session\Session; | ||
8 | use Symfony\Component\HttpFoundation\Session\Storage\MockArraySessionStorage; | ||
9 | use GuzzleHttp\Client; | 7 | use GuzzleHttp\Client; |
10 | use GuzzleHttp\Subscriber\Mock; | 8 | use GuzzleHttp\Subscriber\Mock; |
11 | use GuzzleHttp\Message\Response; | 9 | use GuzzleHttp\Message\Response; |
12 | use GuzzleHttp\Stream\Stream; | 10 | use GuzzleHttp\Stream\Stream; |
11 | use Monolog\Logger; | ||
12 | use Monolog\Handler\TestHandler; | ||
13 | |||
14 | class PocketImportMock extends PocketImport | ||
15 | { | ||
16 | public function getAccessToken() | ||
17 | { | ||
18 | return $this->accessToken; | ||
19 | } | ||
20 | } | ||
13 | 21 | ||
14 | class PocketImportTest extends \PHPUnit_Framework_TestCase | 22 | class PocketImportTest extends \PHPUnit_Framework_TestCase |
15 | { | 23 | { |
16 | protected $token; | 24 | protected $token; |
17 | protected $user; | 25 | protected $user; |
18 | protected $session; | ||
19 | protected $em; | 26 | protected $em; |
27 | protected $contentProxy; | ||
28 | protected $logHandler; | ||
20 | 29 | ||
21 | private function getPocketImport($consumerKey = 'ConsumerKey') | 30 | private function getPocketImport($consumerKey = 'ConsumerKey') |
22 | { | 31 | { |
@@ -30,6 +39,10 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase | |||
30 | ->disableOriginalConstructor() | 39 | ->disableOriginalConstructor() |
31 | ->getMock(); | 40 | ->getMock(); |
32 | 41 | ||
42 | $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') | ||
43 | ->disableOriginalConstructor() | ||
44 | ->getMock(); | ||
45 | |||
33 | $token->expects($this->once()) | 46 | $token->expects($this->once()) |
34 | ->method('getUser') | 47 | ->method('getUser') |
35 | ->willReturn($this->user); | 48 | ->willReturn($this->user); |
@@ -38,18 +51,22 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase | |||
38 | ->method('getToken') | 51 | ->method('getToken') |
39 | ->willReturn($token); | 52 | ->willReturn($token); |
40 | 53 | ||
41 | $this->session = new Session(new MockArraySessionStorage()); | ||
42 | |||
43 | $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') | 54 | $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') |
44 | ->disableOriginalConstructor() | 55 | ->disableOriginalConstructor() |
45 | ->getMock(); | 56 | ->getMock(); |
46 | 57 | ||
47 | return new PocketImport( | 58 | $pocket = new PocketImportMock( |
48 | $this->tokenStorage, | 59 | $this->tokenStorage, |
49 | $this->session, | ||
50 | $this->em, | 60 | $this->em, |
61 | $this->contentProxy, | ||
51 | $consumerKey | 62 | $consumerKey |
52 | ); | 63 | ); |
64 | |||
65 | $this->logHandler = new TestHandler(); | ||
66 | $logger = new Logger('test', array($this->logHandler)); | ||
67 | $pocket->setLogger($logger); | ||
68 | |||
69 | return $pocket; | ||
53 | } | 70 | } |
54 | 71 | ||
55 | public function testInit() | 72 | public function testInit() |
@@ -65,7 +82,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase | |||
65 | $client = new Client(); | 82 | $client = new Client(); |
66 | 83 | ||
67 | $mock = new Mock([ | 84 | $mock = new Mock([ |
68 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['code' => 'wunderbar']))), | 85 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['code' => 'wunderbar_code']))), |
69 | ]); | 86 | ]); |
70 | 87 | ||
71 | $client->getEmitter()->attach($mock); | 88 | $client->getEmitter()->attach($mock); |
@@ -73,10 +90,31 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase | |||
73 | $pocketImport = $this->getPocketImport(); | 90 | $pocketImport = $this->getPocketImport(); |
74 | $pocketImport->setClient($client); | 91 | $pocketImport->setClient($client); |
75 | 92 | ||
76 | $url = $pocketImport->oAuthRequest('http://0.0.0.0./redirect', 'http://0.0.0.0./callback'); | 93 | $code = $pocketImport->getRequestToken('http://0.0.0.0/redirect'); |
77 | 94 | ||
78 | $this->assertEquals('https://getpocket.com/auth/authorize?request_token=wunderbar&redirect_uri=http://0.0.0.0./callback', $url); | 95 | $this->assertEquals('wunderbar_code', $code); |
79 | $this->assertEquals('wunderbar', $this->session->get('pocketCode')); | 96 | } |
97 | |||
98 | public function testOAuthRequestBadResponse() | ||
99 | { | ||
100 | $client = new Client(); | ||
101 | |||
102 | $mock = new Mock([ | ||
103 | new Response(403), | ||
104 | ]); | ||
105 | |||
106 | $client->getEmitter()->attach($mock); | ||
107 | |||
108 | $pocketImport = $this->getPocketImport(); | ||
109 | $pocketImport->setClient($client); | ||
110 | |||
111 | $code = $pocketImport->getRequestToken('http://0.0.0.0/redirect'); | ||
112 | |||
113 | $this->assertFalse($code); | ||
114 | |||
115 | $records = $this->logHandler->getRecords(); | ||
116 | $this->assertContains('PocketImport: Failed to request token', $records[0]['message']); | ||
117 | $this->assertEquals('ERROR', $records[0]['level_name']); | ||
80 | } | 118 | } |
81 | 119 | ||
82 | public function testOAuthAuthorize() | 120 | public function testOAuthAuthorize() |
@@ -84,7 +122,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase | |||
84 | $client = new Client(); | 122 | $client = new Client(); |
85 | 123 | ||
86 | $mock = new Mock([ | 124 | $mock = new Mock([ |
87 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar']))), | 125 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))), |
88 | ]); | 126 | ]); |
89 | 127 | ||
90 | $client->getEmitter()->attach($mock); | 128 | $client->getEmitter()->attach($mock); |
@@ -92,26 +130,182 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase | |||
92 | $pocketImport = $this->getPocketImport(); | 130 | $pocketImport = $this->getPocketImport(); |
93 | $pocketImport->setClient($client); | 131 | $pocketImport->setClient($client); |
94 | 132 | ||
95 | $accessToken = $pocketImport->oAuthAuthorize(); | 133 | $res = $pocketImport->authorize('wunderbar_code'); |
96 | 134 | ||
97 | $this->assertEquals('wunderbar', $accessToken); | 135 | $this->assertTrue($res); |
136 | $this->assertEquals('wunderbar_token', $pocketImport->getAccessToken()); | ||
98 | } | 137 | } |
99 | 138 | ||
139 | public function testOAuthAuthorizeBadResponse() | ||
140 | { | ||
141 | $client = new Client(); | ||
142 | |||
143 | $mock = new Mock([ | ||
144 | new Response(403), | ||
145 | ]); | ||
146 | |||
147 | $client->getEmitter()->attach($mock); | ||
148 | |||
149 | $pocketImport = $this->getPocketImport(); | ||
150 | $pocketImport->setClient($client); | ||
151 | |||
152 | $res = $pocketImport->authorize('wunderbar_code'); | ||
153 | |||
154 | $this->assertFalse($res); | ||
155 | |||
156 | $records = $this->logHandler->getRecords(); | ||
157 | $this->assertContains('PocketImport: Failed to authorize client', $records[0]['message']); | ||
158 | $this->assertEquals('ERROR', $records[0]['level_name']); | ||
159 | } | ||
160 | |||
161 | /** | ||
162 | * Will sample results from https://getpocket.com/developer/docs/v3/retrieve. | ||
163 | */ | ||
100 | public function testImport() | 164 | public function testImport() |
101 | { | 165 | { |
102 | $client = new Client(); | 166 | $client = new Client(); |
103 | 167 | ||
104 | $mock = new Mock([ | 168 | $mock = new Mock([ |
105 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['list' => []]))), | 169 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))), |
170 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(' | ||
171 | { | ||
172 | "status": 1, | ||
173 | "list": { | ||
174 | "229279689": { | ||
175 | "item_id": "229279689", | ||
176 | "resolved_id": "229279689", | ||
177 | "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", | ||
178 | "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", | ||
179 | "favorite": "1", | ||
180 | "status": "1", | ||
181 | "resolved_title": "The Massive Ryder Cup Preview", | ||
182 | "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", | ||
183 | "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", | ||
184 | "is_article": "1", | ||
185 | "has_video": "1", | ||
186 | "has_image": "1", | ||
187 | "word_count": "3197", | ||
188 | "images": { | ||
189 | "1": { | ||
190 | "item_id": "229279689", | ||
191 | "image_id": "1", | ||
192 | "src": "http://a.espncdn.com/combiner/i?img=/photo/2012/0927/grant_g_ryder_cr_640.jpg&w=640&h=360", | ||
193 | "width": "0", | ||
194 | "height": "0", | ||
195 | "credit": "Jamie Squire/Getty Images", | ||
196 | "caption": "" | ||
197 | } | ||
198 | }, | ||
199 | "videos": { | ||
200 | "1": { | ||
201 | "item_id": "229279689", | ||
202 | "video_id": "1", | ||
203 | "src": "http://www.youtube.com/v/Er34PbFkVGk?version=3&hl=en_US&rel=0", | ||
204 | "width": "420", | ||
205 | "height": "315", | ||
206 | "type": "1", | ||
207 | "vid": "Er34PbFkVGk" | ||
208 | } | ||
209 | }, | ||
210 | "tags": { | ||
211 | "grantland": { | ||
212 | "item_id": "1147652870", | ||
213 | "tag": "grantland" | ||
214 | }, | ||
215 | "Ryder Cup": { | ||
216 | "item_id": "1147652870", | ||
217 | "tag": "Ryder Cup" | ||
218 | } | ||
219 | } | ||
220 | }, | ||
221 | "229279690": { | ||
222 | "item_id": "229279689", | ||
223 | "resolved_id": "229279689", | ||
224 | "given_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", | ||
225 | "given_title": "The Massive Ryder Cup Preview - The Triangle Blog - Grantland", | ||
226 | "favorite": "1", | ||
227 | "status": "1", | ||
228 | "resolved_title": "The Massive Ryder Cup Preview", | ||
229 | "resolved_url": "http://www.grantland.com/blog/the-triangle/post/_/id/38347/ryder-cup-preview", | ||
230 | "excerpt": "The list of things I love about the Ryder Cup is so long that it could fill a (tedious) novel, and golf fans can probably guess most of them.", | ||
231 | "is_article": "1", | ||
232 | "has_video": "0", | ||
233 | "has_image": "0", | ||
234 | "word_count": "3197" | ||
235 | } | ||
236 | } | ||
237 | } | ||
238 | ')), | ||
239 | ]); | ||
240 | |||
241 | $client->getEmitter()->attach($mock); | ||
242 | |||
243 | $pocketImport = $this->getPocketImport(); | ||
244 | |||
245 | $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') | ||
246 | ->disableOriginalConstructor() | ||
247 | ->getMock(); | ||
248 | |||
249 | $entryRepo->expects($this->exactly(2)) | ||
250 | ->method('existByUrlAndUserId') | ||
251 | ->will($this->onConsecutiveCalls(false, true)); | ||
252 | |||
253 | $tag = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Tag') | ||
254 | ->disableOriginalConstructor() | ||
255 | ->getMock(); | ||
256 | |||
257 | $tagRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\TagRepository') | ||
258 | ->disableOriginalConstructor() | ||
259 | ->getMock(); | ||
260 | |||
261 | $tagRepo->expects($this->exactly(2)) | ||
262 | ->method('findOneByLabelAndUserId') | ||
263 | ->will($this->onConsecutiveCalls(false, $tag)); | ||
264 | |||
265 | $this->em | ||
266 | ->expects($this->any()) | ||
267 | ->method('getRepository') | ||
268 | ->will($this->onConsecutiveCalls($entryRepo, $tagRepo, $tagRepo, $entryRepo)); | ||
269 | |||
270 | $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') | ||
271 | ->disableOriginalConstructor() | ||
272 | ->getMock(); | ||
273 | |||
274 | $this->contentProxy | ||
275 | ->expects($this->once()) | ||
276 | ->method('updateEntry') | ||
277 | ->willReturn($entry); | ||
278 | |||
279 | $pocketImport->setClient($client); | ||
280 | $pocketImport->authorize('wunderbar_code'); | ||
281 | |||
282 | $res = $pocketImport->import(); | ||
283 | |||
284 | $this->assertTrue($res); | ||
285 | $this->assertEquals(['skipped' => 1, 'imported' => 1], $pocketImport->getSummary()); | ||
286 | } | ||
287 | |||
288 | public function testImportBadResponse() | ||
289 | { | ||
290 | $client = new Client(); | ||
291 | |||
292 | $mock = new Mock([ | ||
293 | new Response(200, ['Content-Type' => 'application/json'], Stream::factory(json_encode(['access_token' => 'wunderbar_token']))), | ||
294 | new Response(403), | ||
106 | ]); | 295 | ]); |
107 | 296 | ||
108 | $client->getEmitter()->attach($mock); | 297 | $client->getEmitter()->attach($mock); |
109 | 298 | ||
110 | $pocketImport = $this->getPocketImport(); | 299 | $pocketImport = $this->getPocketImport(); |
111 | $pocketImport->setClient($client); | 300 | $pocketImport->setClient($client); |
301 | $pocketImport->authorize('wunderbar_code'); | ||
302 | |||
303 | $res = $pocketImport->import(); | ||
112 | 304 | ||
113 | $pocketImport->import('wunderbar'); | 305 | $this->assertFalse($res); |
114 | 306 | ||
115 | $this->assertEquals('0 entries imported, 0 already saved.', $this->session->getFlashBag()->get('notice')[0]); | 307 | $records = $this->logHandler->getRecords(); |
308 | $this->assertContains('PocketImport: Failed to import', $records[0]['message']); | ||
309 | $this->assertEquals('ERROR', $records[0]['level_name']); | ||
116 | } | 310 | } |
117 | } | 311 | } |