aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorJérémy Benoist <j0k3r@users.noreply.github.com>2017-06-02 11:26:37 +0200
committerGitHub <noreply@github.com>2017-06-02 11:26:37 +0200
commita687c8d915276eee0c0494156700f7d0c0606735 (patch)
tree23178dc2407aa1b926b79281c34d8e05b6318d3b
parent14b8a7c950147d32d7c9782832b87bf2b18b4fd7 (diff)
parentf5924e954730efdb7b9fadf23c0b73b3f5a0a434 (diff)
downloadwallabag-a687c8d915276eee0c0494156700f7d0c0606735.tar.gz
wallabag-a687c8d915276eee0c0494156700f7d0c0606735.tar.zst
wallabag-a687c8d915276eee0c0494156700f7d0c0606735.zip
Merge pull request #2708 from jcharaoui/import-disablecontentupdate
Import disableContentUpdate
-rw-r--r--app/config/config.yml1
-rw-r--r--src/Wallabag/ApiBundle/Controller/EntryRestController.php49
-rw-r--r--src/Wallabag/CoreBundle/Controller/EntryController.php11
-rw-r--r--src/Wallabag/CoreBundle/DependencyInjection/Configuration.php2
-rw-r--r--src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php1
-rw-r--r--src/Wallabag/CoreBundle/Helper/ContentProxy.php48
-rw-r--r--src/Wallabag/CoreBundle/Resources/config/services.yml1
-rw-r--r--src/Wallabag/ImportBundle/Command/ImportCommand.php9
-rw-r--r--src/Wallabag/ImportBundle/Import/AbstractImport.php22
-rw-r--r--src/Wallabag/ImportBundle/Import/BrowserImport.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/InstapaperImport.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/PinboardImport.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/PocketImport.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/ReadabilityImport.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagImport.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV1Import.php18
-rw-r--r--src/Wallabag/ImportBundle/Resources/config/services.yml2
-rw-r--r--tests/Wallabag/CoreBundle/Controller/TagControllerTest.php2
-rw-r--r--tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php74
-rw-r--r--tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php11
20 files changed, 160 insertions, 103 deletions
diff --git a/app/config/config.yml b/app/config/config.yml
index 9792616e..04f8547d 100644
--- a/app/config/config.yml
+++ b/app/config/config.yml
@@ -58,6 +58,7 @@ wallabag_core:
58 cache_lifetime: 10 58 cache_lifetime: 10
59 action_mark_as_read: 1 59 action_mark_as_read: 1
60 list_mode: 0 60 list_mode: 0
61 fetching_error_message_title: 'No title found'
61 fetching_error_message: | 62 fetching_error_message: |
62 wallabag can't retrieve contents for this article. Please <a href="http://doc.wallabag.org/en/user/errors_during_fetching.html#how-can-i-help-to-fix-that">troubleshoot this issue</a>. 63 wallabag can't retrieve contents for this article. Please <a href="http://doc.wallabag.org/en/user/errors_during_fetching.html#how-can-i-help-to-fix-that">troubleshoot this issue</a>.
63 api_limit_mass_actions: 10 64 api_limit_mass_actions: 10
diff --git a/src/Wallabag/ApiBundle/Controller/EntryRestController.php b/src/Wallabag/ApiBundle/Controller/EntryRestController.php
index c3ba1858..93c8157e 100644
--- a/src/Wallabag/ApiBundle/Controller/EntryRestController.php
+++ b/src/Wallabag/ApiBundle/Controller/EntryRestController.php
@@ -231,7 +231,6 @@ class EntryRestController extends WallabagRestController
231 $this->validateAuthentication(); 231 $this->validateAuthentication();
232 232
233 $urls = json_decode($request->query->get('urls', [])); 233 $urls = json_decode($request->query->get('urls', []));
234 $results = [];
235 234
236 $limit = $this->container->getParameter('wallabag_core.api_limit_mass_actions'); 235 $limit = $this->container->getParameter('wallabag_core.api_limit_mass_actions');
237 236
@@ -239,32 +238,34 @@ class EntryRestController extends WallabagRestController
239 throw new HttpException(400, 'API limit reached'); 238 throw new HttpException(400, 'API limit reached');
240 } 239 }
241 240
241 $results = [];
242 if (empty($urls)) {
243 return $this->sendResponse($results);
244 }
245
242 // handle multiple urls 246 // handle multiple urls
243 if (!empty($urls)) { 247 foreach ($urls as $key => $url) {
244 foreach ($urls as $key => $url) { 248 $entry = $this->get('wallabag_core.entry_repository')->findByUrlAndUserId(
245 $entry = $this->get('wallabag_core.entry_repository')->findByUrlAndUserId( 249 $url,
246 $url, 250 $this->getUser()->getId()
247 $this->getUser()->getId() 251 );
248 );
249
250 $results[$key]['url'] = $url;
251
252 if (false === $entry) {
253 $entry = $this->get('wallabag_core.content_proxy')->updateEntry(
254 new Entry($this->getUser()),
255 $url
256 );
257 }
258 252
259 $em = $this->getDoctrine()->getManager(); 253 $results[$key]['url'] = $url;
260 $em->persist($entry);
261 $em->flush();
262 254
263 $results[$key]['entry'] = $entry instanceof Entry ? $entry->getId() : false; 255 if (false === $entry) {
256 $entry = new Entry($this->getUser());
264 257
265 // entry saved, dispatch event about it! 258 $this->get('wallabag_core.content_proxy')->updateEntry($entry, $url);
266 $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
267 } 259 }
260
261 $em = $this->getDoctrine()->getManager();
262 $em->persist($entry);
263 $em->flush();
264
265 $results[$key]['entry'] = $entry instanceof Entry ? $entry->getId() : false;
266
267 // entry saved, dispatch event about it!
268 $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
268 } 269 }
269 270
270 return $this->sendResponse($results); 271 return $this->sendResponse($results);
@@ -315,7 +316,7 @@ class EntryRestController extends WallabagRestController
315 } 316 }
316 317
317 try { 318 try {
318 $entry = $this->get('wallabag_core.content_proxy')->updateEntry( 319 $this->get('wallabag_core.content_proxy')->updateEntry(
319 $entry, 320 $entry,
320 $url, 321 $url,
321 [ 322 [
@@ -428,7 +429,7 @@ class EntryRestController extends WallabagRestController
428 $this->validateUserAccess($entry->getUser()->getId()); 429 $this->validateUserAccess($entry->getUser()->getId());
429 430
430 try { 431 try {
431 $entry = $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl()); 432 $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl());
432 } catch (\Exception $e) { 433 } catch (\Exception $e) {
433 $this->get('logger')->error('Error while saving an entry', [ 434 $this->get('logger')->error('Error while saving an entry', [
434 'exception' => $e, 435 'exception' => $e,
diff --git a/src/Wallabag/CoreBundle/Controller/EntryController.php b/src/Wallabag/CoreBundle/Controller/EntryController.php
index 9fe3e693..fafa49f1 100644
--- a/src/Wallabag/CoreBundle/Controller/EntryController.php
+++ b/src/Wallabag/CoreBundle/Controller/EntryController.php
@@ -53,22 +53,17 @@ class EntryController extends Controller
53 53
54 /** 54 /**
55 * Fetch content and update entry. 55 * Fetch content and update entry.
56 * In case it fails, entry will return to avod loosing the data. 56 * In case it fails, $entry->getContent will return an error message.
57 * 57 *
58 * @param Entry $entry 58 * @param Entry $entry
59 * @param string $prefixMessage Should be the translation key: entry_saved or entry_reloaded 59 * @param string $prefixMessage Should be the translation key: entry_saved or entry_reloaded
60 *
61 * @return Entry
62 */ 60 */
63 private function updateEntry(Entry $entry, $prefixMessage = 'entry_saved') 61 private function updateEntry(Entry $entry, $prefixMessage = 'entry_saved')
64 { 62 {
65 // put default title in case of fetching content failed
66 $entry->setTitle('No title found');
67
68 $message = 'flashes.entry.notice.'.$prefixMessage; 63 $message = 'flashes.entry.notice.'.$prefixMessage;
69 64
70 try { 65 try {
71 $entry = $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl()); 66 $this->get('wallabag_core.content_proxy')->updateEntry($entry, $entry->getUrl());
72 } catch (\Exception $e) { 67 } catch (\Exception $e) {
73 $this->get('logger')->error('Error while saving an entry', [ 68 $this->get('logger')->error('Error while saving an entry', [
74 'exception' => $e, 69 'exception' => $e,
@@ -79,8 +74,6 @@ class EntryController extends Controller
79 } 74 }
80 75
81 $this->get('session')->getFlashBag()->add('notice', $message); 76 $this->get('session')->getFlashBag()->add('notice', $message);
82
83 return $entry;
84 } 77 }
85 78
86 /** 79 /**
diff --git a/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php b/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php
index 75b37729..8b5b5744 100644
--- a/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php
+++ b/src/Wallabag/CoreBundle/DependencyInjection/Configuration.php
@@ -41,6 +41,8 @@ class Configuration implements ConfigurationInterface
41 ->end() 41 ->end()
42 ->scalarNode('fetching_error_message') 42 ->scalarNode('fetching_error_message')
43 ->end() 43 ->end()
44 ->scalarNode('fetching_error_message_title')
45 ->end()
44 ->scalarNode('action_mark_as_read') 46 ->scalarNode('action_mark_as_read')
45 ->defaultValue(1) 47 ->defaultValue(1)
46 ->end() 48 ->end()
diff --git a/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php b/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php
index c075c19f..a2a703cb 100644
--- a/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php
+++ b/src/Wallabag/CoreBundle/DependencyInjection/WallabagCoreExtension.php
@@ -26,6 +26,7 @@ class WallabagCoreExtension extends Extension
26 $container->setParameter('wallabag_core.action_mark_as_read', $config['action_mark_as_read']); 26 $container->setParameter('wallabag_core.action_mark_as_read', $config['action_mark_as_read']);
27 $container->setParameter('wallabag_core.list_mode', $config['list_mode']); 27 $container->setParameter('wallabag_core.list_mode', $config['list_mode']);
28 $container->setParameter('wallabag_core.fetching_error_message', $config['fetching_error_message']); 28 $container->setParameter('wallabag_core.fetching_error_message', $config['fetching_error_message']);
29 $container->setParameter('wallabag_core.fetching_error_message_title', $config['fetching_error_message_title']);
29 $container->setParameter('wallabag_core.api_limit_mass_actions', $config['api_limit_mass_actions']); 30 $container->setParameter('wallabag_core.api_limit_mass_actions', $config['api_limit_mass_actions']);
30 31
31 $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); 32 $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php
index 8ba77ca9..bfaa1976 100644
--- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php
+++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php
@@ -31,27 +31,20 @@ class ContentProxy
31 } 31 }
32 32
33 /** 33 /**
34 * Fetch content using graby and hydrate given $entry with results information. 34 * Update entry using either fetched or provided content.
35 * In case we couldn't find content, we'll try to use Open Graph data.
36 * 35 *
37 * We can also force the content, in case of an import from the v1 for example, so the function won't 36 * @param Entry $entry Entry to update
38 * fetch the content from the website but rather use information given with the $content parameter. 37 * @param string $url Url of the content
39 * 38 * @param array $content Array with content provided for import with AT LEAST keys title, html, url to skip the fetchContent from the url
40 * @param Entry $entry Entry to update 39 * @param bool $disableContentUpdate Whether to skip trying to fetch content using Graby
41 * @param string $url Url to grab content for
42 * @param array $content An array with AT LEAST keys title, html, url to skip the fetchContent from the url
43 *
44 * @return Entry
45 */ 40 */
46 public function updateEntry(Entry $entry, $url, array $content = []) 41 public function updateEntry(Entry $entry, $url, array $content = [], $disableContentUpdate = false)
47 { 42 {
48 // ensure content is a bit cleaned up
49 if (!empty($content['html'])) { 43 if (!empty($content['html'])) {
50 $content['html'] = $this->graby->cleanupHtml($content['html'], $url); 44 $content['html'] = $this->graby->cleanupHtml($content['html'], $url);
51 } 45 }
52 46
53 // do we have to fetch the content or the provided one is ok? 47 if ((empty($content) || false === $this->validateContent($content)) && false === $disableContentUpdate) {
54 if (empty($content) || false === $this->validateContent($content)) {
55 $fetchedContent = $this->graby->fetchContent($url); 48 $fetchedContent = $this->graby->fetchContent($url);
56 49
57 // when content is imported, we have information in $content 50 // when content is imported, we have information in $content
@@ -61,6 +54,22 @@ class ContentProxy
61 } 54 }
62 } 55 }
63 56
57 // be sure to keep the url in case of error
58 // so we'll be able to refetch it in the future
59 $content['url'] = !empty($content['url']) ? $content['url'] : $url;
60
61 $this->stockEntry($entry, $content);
62 }
63
64 /**
65 * Stock entry with fetched or imported content.
66 * Will fall back to OpenGraph data if available.
67 *
68 * @param Entry $entry Entry to stock
69 * @param array $content Array with at least title, url & html
70 */
71 private function stockEntry(Entry $entry, array $content)
72 {
64 $title = $content['title']; 73 $title = $content['title'];
65 if (!$title && !empty($content['open_graph']['og_title'])) { 74 if (!$title && !empty($content['open_graph']['og_title'])) {
66 $title = $content['open_graph']['og_title']; 75 $title = $content['open_graph']['og_title'];
@@ -76,7 +85,7 @@ class ContentProxy
76 } 85 }
77 } 86 }
78 87
79 $entry->setUrl($content['url'] ?: $url); 88 $entry->setUrl($content['url']);
80 $entry->setTitle($title); 89 $entry->setTitle($title);
81 $entry->setContent($html); 90 $entry->setContent($html);
82 $entry->setHttpStatus(isset($content['status']) ? $content['status'] : ''); 91 $entry->setHttpStatus(isset($content['status']) ? $content['status'] : '');
@@ -92,7 +101,7 @@ class ContentProxy
92 try { 101 try {
93 $entry->setPublishedAt(new \DateTime($date)); 102 $entry->setPublishedAt(new \DateTime($date));
94 } catch (\Exception $e) { 103 } catch (\Exception $e) {
95 $this->logger->warning('Error while defining date', ['e' => $e, 'url' => $url, 'date' => $content['date']]); 104 $this->logger->warning('Error while defining date', ['e' => $e, 'url' => $content['url'], 'date' => $content['date']]);
96 } 105 }
97 } 106 }
98 107
@@ -126,17 +135,14 @@ class ContentProxy
126 $this->tagger->tag($entry); 135 $this->tagger->tag($entry);
127 } catch (\Exception $e) { 136 } catch (\Exception $e) {
128 $this->logger->error('Error while trying to automatically tag an entry.', [ 137 $this->logger->error('Error while trying to automatically tag an entry.', [
129 'entry_url' => $url, 138 'entry_url' => $content['url'],
130 'error_msg' => $e->getMessage(), 139 'error_msg' => $e->getMessage(),
131 ]); 140 ]);
132 } 141 }
133
134 return $entry;
135 } 142 }
136 143
137 /** 144 /**
138 * Validate that the given content as enough value to be used 145 * Validate that the given content has at least a title, an html and a url.
139 * instead of fetch the content from the url.
140 * 146 *
141 * @param array $content 147 * @param array $content
142 * 148 *
diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml
index a68b2fdc..a9b0d2d5 100644
--- a/src/Wallabag/CoreBundle/Resources/config/services.yml
+++ b/src/Wallabag/CoreBundle/Resources/config/services.yml
@@ -41,6 +41,7 @@ services:
41 arguments: 41 arguments:
42 - 42 -
43 error_message: '%wallabag_core.fetching_error_message%' 43 error_message: '%wallabag_core.fetching_error_message%'
44 error_message_title: '%wallabag_core.fetching_error_message_title%'
44 - "@wallabag_core.guzzle.http_client" 45 - "@wallabag_core.guzzle.http_client"
45 - "@wallabag_core.graby.config_builder" 46 - "@wallabag_core.graby.config_builder"
46 calls: 47 calls:
diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php
index ce72837a..5f1ab0af 100644
--- a/src/Wallabag/ImportBundle/Command/ImportCommand.php
+++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php
@@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Command;
5use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; 5use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
6use Symfony\Component\Config\Definition\Exception\Exception; 6use Symfony\Component\Config\Definition\Exception\Exception;
7use Symfony\Component\Console\Input\InputArgument; 7use Symfony\Component\Console\Input\InputArgument;
8use Symfony\Component\Console\Input\InputOption;
8use Symfony\Component\Console\Input\InputInterface; 9use Symfony\Component\Console\Input\InputInterface;
9use Symfony\Component\Console\Output\OutputInterface; 10use Symfony\Component\Console\Output\OutputInterface;
10 11
@@ -17,9 +18,10 @@ class ImportCommand extends ContainerAwareCommand
17 ->setDescription('Import entries from a JSON export') 18 ->setDescription('Import entries from a JSON export')
18 ->addArgument('username', InputArgument::REQUIRED, 'User to populate') 19 ->addArgument('username', InputArgument::REQUIRED, 'User to populate')
19 ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') 20 ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file')
20 ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, readability, firefox or chrome', 'v1') 21 ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, readability, firefox or chrome', 'v1')
21 ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) 22 ->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false)
22 ->addOption('useUserId', null, InputArgument::OPTIONAL, 'Use user id instead of username to find account', false) 23 ->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account')
24 ->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL')
23 ; 25 ;
24 } 26 }
25 27
@@ -69,6 +71,7 @@ class ImportCommand extends ContainerAwareCommand
69 } 71 }
70 72
71 $import->setMarkAsRead($input->getOption('markAsRead')); 73 $import->setMarkAsRead($input->getOption('markAsRead'));
74 $import->setDisableContentUpdate($input->getOption('disableContentUpdate'));
72 $import->setUser($user); 75 $import->setUser($user);
73 76
74 $res = $import 77 $res = $import
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index a61388c0..9b624296 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -24,6 +24,7 @@ abstract class AbstractImport implements ImportInterface
24 protected $producer; 24 protected $producer;
25 protected $user; 25 protected $user;
26 protected $markAsRead; 26 protected $markAsRead;
27 protected $disableContentUpdate = false;
27 protected $skippedEntries = 0; 28 protected $skippedEntries = 0;
28 protected $importedEntries = 0; 29 protected $importedEntries = 0;
29 protected $queuedEntries = 0; 30 protected $queuedEntries = 0;
@@ -85,21 +86,34 @@ abstract class AbstractImport implements ImportInterface
85 } 86 }
86 87
87 /** 88 /**
89 * Set whether articles should be fetched for updated content.
90 *
91 * @param bool $disableContentUpdate
92 */
93 public function setDisableContentUpdate($disableContentUpdate)
94 {
95 $this->disableContentUpdate = $disableContentUpdate;
96
97 return $this;
98 }
99
100 /**
88 * Fetch content from the ContentProxy (using graby). 101 * Fetch content from the ContentProxy (using graby).
89 * If it fails return the given entry to be saved in all case (to avoid user to loose the content). 102 * If it fails return the given entry to be saved in all case (to avoid user to loose the content).
90 * 103 *
91 * @param Entry $entry Entry to update 104 * @param Entry $entry Entry to update
92 * @param string $url Url to grab content for 105 * @param string $url Url to grab content for
93 * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url 106 * @param array $content An array with AT LEAST keys title, html, url, language & content_type to skip the fetchContent from the url
94 *
95 * @return Entry
96 */ 107 */
97 protected function fetchContent(Entry $entry, $url, array $content = []) 108 protected function fetchContent(Entry $entry, $url, array $content = [])
98 { 109 {
99 try { 110 try {
100 return $this->contentProxy->updateEntry($entry, $url, $content); 111 $this->contentProxy->updateEntry($entry, $url, $content, $this->disableContentUpdate);
101 } catch (\Exception $e) { 112 } catch (\Exception $e) {
102 return $entry; 113 $this->logger->error('Error trying to import an entry.', [
114 'entry_url' => $url,
115 'error_msg' => $e->getMessage(),
116 ]);
103 } 117 }
104 } 118 }
105 119
diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php
index ef0eeb7e..71e65e59 100644
--- a/src/Wallabag/ImportBundle/Import/BrowserImport.php
+++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php
@@ -201,7 +201,7 @@ abstract class BrowserImport extends AbstractImport
201 $entry->setTitle($data['title']); 201 $entry->setTitle($data['title']);
202 202
203 // update entry with content (in case fetching failed, the given entry will be return) 203 // update entry with content (in case fetching failed, the given entry will be return)
204 $entry = $this->fetchContent($entry, $data['url'], $data); 204 $this->fetchContent($entry, $data['url'], $data);
205 205
206 if (array_key_exists('tags', $data)) { 206 if (array_key_exists('tags', $data)) {
207 $this->tagsAssigner->assignTagsToEntry( 207 $this->tagsAssigner->assignTagsToEntry(
diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
index c8e0cd5b..3aa12f6f 100644
--- a/src/Wallabag/ImportBundle/Import/InstapaperImport.php
+++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
@@ -125,7 +125,7 @@ class InstapaperImport extends AbstractImport
125 $entry->setTitle($importedEntry['title']); 125 $entry->setTitle($importedEntry['title']);
126 126
127 // update entry with content (in case fetching failed, the given entry will be return) 127 // update entry with content (in case fetching failed, the given entry will be return)
128 $entry = $this->fetchContent($entry, $importedEntry['url'], $importedEntry); 128 $this->fetchContent($entry, $importedEntry['url'], $importedEntry);
129 129
130 if (!empty($importedEntry['tags'])) { 130 if (!empty($importedEntry['tags'])) {
131 $this->tagsAssigner->assignTagsToEntry( 131 $this->tagsAssigner->assignTagsToEntry(
diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php
index 489b9257..110b0464 100644
--- a/src/Wallabag/ImportBundle/Import/PinboardImport.php
+++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php
@@ -109,7 +109,7 @@ class PinboardImport extends AbstractImport
109 $entry->setTitle($data['title']); 109 $entry->setTitle($data['title']);
110 110
111 // update entry with content (in case fetching failed, the given entry will be return) 111 // update entry with content (in case fetching failed, the given entry will be return)
112 $entry = $this->fetchContent($entry, $data['url'], $data); 112 $this->fetchContent($entry, $data['url'], $data);
113 113
114 if (!empty($data['tags'])) { 114 if (!empty($data['tags'])) {
115 $this->tagsAssigner->assignTagsToEntry( 115 $this->tagsAssigner->assignTagsToEntry(
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 8835161b..c1d5b6da 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -192,7 +192,7 @@ class PocketImport extends AbstractImport
192 $entry->setUrl($url); 192 $entry->setUrl($url);
193 193
194 // update entry with content (in case fetching failed, the given entry will be return) 194 // update entry with content (in case fetching failed, the given entry will be return)
195 $entry = $this->fetchContent($entry, $url); 195 $this->fetchContent($entry, $url);
196 196
197 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted 197 // 0, 1, 2 - 1 if the item is archived - 2 if the item should be deleted
198 $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead); 198 $entry->setArchived($importedEntry['status'] == 1 || $this->markAsRead);
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
index de320d23..002b27f4 100644
--- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -109,7 +109,7 @@ class ReadabilityImport extends AbstractImport
109 $entry->setTitle($data['title']); 109 $entry->setTitle($data['title']);
110 110
111 // update entry with content (in case fetching failed, the given entry will be return) 111 // update entry with content (in case fetching failed, the given entry will be return)
112 $entry = $this->fetchContent($entry, $data['url'], $data); 112 $this->fetchContent($entry, $data['url'], $data);
113 113
114 $entry->setArchived($data['is_archived']); 114 $entry->setArchived($data['is_archived']);
115 $entry->setStarred($data['is_starred']); 115 $entry->setStarred($data['is_starred']);
diff --git a/src/Wallabag/ImportBundle/Import/WallabagImport.php b/src/Wallabag/ImportBundle/Import/WallabagImport.php
index 0e5382cf..c64ccd64 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagImport.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagImport.php
@@ -108,7 +108,7 @@ abstract class WallabagImport extends AbstractImport
108 $entry->setTitle($data['title']); 108 $entry->setTitle($data['title']);
109 109
110 // update entry with content (in case fetching failed, the given entry will be return) 110 // update entry with content (in case fetching failed, the given entry will be return)
111 $entry = $this->fetchContent($entry, $data['url'], $data); 111 $this->fetchContent($entry, $data['url'], $data);
112 112
113 if (array_key_exists('tags', $data)) { 113 if (array_key_exists('tags', $data)) {
114 $this->tagsAssigner->assignTagsToEntry( 114 $this->tagsAssigner->assignTagsToEntry(
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
index 59e3ce02..1f0df646 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
@@ -4,6 +4,17 @@ namespace Wallabag\ImportBundle\Import;
4 4
5class WallabagV1Import extends WallabagImport 5class WallabagV1Import extends WallabagImport
6{ 6{
7 protected $fetchingErrorMessage;
8 protected $fetchingErrorMessageTitle;
9
10 public function __construct($em, $contentProxy, $tagsAssigner, $eventDispatcher, $fetchingErrorMessageTitle, $fetchingErrorMessage)
11 {
12 $this->fetchingErrorMessageTitle = $fetchingErrorMessageTitle;
13 $this->fetchingErrorMessage = $fetchingErrorMessage;
14
15 parent::__construct($em, $contentProxy, $tagsAssigner, $eventDispatcher);
16 }
17
7 /** 18 /**
8 * {@inheritdoc} 19 * {@inheritdoc}
9 */ 20 */
@@ -43,10 +54,11 @@ class WallabagV1Import extends WallabagImport
43 'created_at' => '', 54 'created_at' => '',
44 ]; 55 ];
45 56
46 // force content to be refreshed in case on bad fetch in the v1 installation 57 // In case of a bad fetch in v1, replace title and content with v2 error strings
58 // If fetching fails again, they will get this instead of the v1 strings
47 if (in_array($entry['title'], $this->untitled)) { 59 if (in_array($entry['title'], $this->untitled)) {
48 $data['title'] = ''; 60 $data['title'] = $this->fetchingErrorMessageTitle;
49 $data['html'] = ''; 61 $data['html'] = $this->fetchingErrorMessage;
50 } 62 }
51 63
52 if (array_key_exists('tags', $entry) && $entry['tags'] != '') { 64 if (array_key_exists('tags', $entry) && $entry['tags'] != '') {
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index 661dc7e1..b224a6a2 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -35,6 +35,8 @@ services:
35 - "@wallabag_core.content_proxy" 35 - "@wallabag_core.content_proxy"
36 - "@wallabag_core.tags_assigner" 36 - "@wallabag_core.tags_assigner"
37 - "@event_dispatcher" 37 - "@event_dispatcher"
38 - "%wallabag_core.fetching_error_message_title%"
39 - "%wallabag_core.fetching_error_message%"
38 calls: 40 calls:
39 - [ setLogger, [ "@logger" ]] 41 - [ setLogger, [ "@logger" ]]
40 tags: 42 tags:
diff --git a/tests/Wallabag/CoreBundle/Controller/TagControllerTest.php b/tests/Wallabag/CoreBundle/Controller/TagControllerTest.php
index f9bf7b87..af1ad7af 100644
--- a/tests/Wallabag/CoreBundle/Controller/TagControllerTest.php
+++ b/tests/Wallabag/CoreBundle/Controller/TagControllerTest.php
@@ -123,7 +123,7 @@ class TagControllerTest extends WallabagCoreTestCase
123 $this->assertEquals(302, $client->getResponse()->getStatusCode()); 123 $this->assertEquals(302, $client->getResponse()->getStatusCode());
124 $this->assertEquals($entryUri, $client->getResponse()->getTargetUrl()); 124 $this->assertEquals($entryUri, $client->getResponse()->getTargetUrl());
125 125
126 // re-retrieve the entry to be sure to get fresh data from database (mostly for tags) 126 // re-retrieve the entry to be sure to get fresh data from database (mostly for tags)
127 $entry = $this->getEntityManager()->getRepository(Entry::class)->find($entry->getId()); 127 $entry = $this->getEntityManager()->getRepository(Entry::class)->find($entry->getId());
128 $this->assertNotContains($this->tagName, $entry->getTags()); 128 $this->assertNotContains($this->tagName, $entry->getTags());
129 129
diff --git a/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php b/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php
index 0c715d90..a3570125 100644
--- a/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php
+++ b/tests/Wallabag/CoreBundle/Helper/ContentProxyTest.php
@@ -3,14 +3,14 @@
3namespace Tests\Wallabag\CoreBundle\Helper; 3namespace Tests\Wallabag\CoreBundle\Helper;
4 4
5use Psr\Log\NullLogger; 5use Psr\Log\NullLogger;
6use Monolog\Logger;
7use Monolog\Handler\TestHandler;
6use Wallabag\CoreBundle\Helper\ContentProxy; 8use Wallabag\CoreBundle\Helper\ContentProxy;
7use Wallabag\CoreBundle\Entity\Entry; 9use Wallabag\CoreBundle\Entity\Entry;
8use Wallabag\CoreBundle\Entity\Tag; 10use Wallabag\CoreBundle\Entity\Tag;
9use Wallabag\UserBundle\Entity\User; 11use Wallabag\UserBundle\Entity\User;
10use Wallabag\CoreBundle\Helper\RuleBasedTagger; 12use Wallabag\CoreBundle\Helper\RuleBasedTagger;
11use Graby\Graby; 13use Graby\Graby;
12use Monolog\Handler\TestHandler;
13use Monolog\Logger;
14 14
15class ContentProxyTest extends \PHPUnit_Framework_TestCase 15class ContentProxyTest extends \PHPUnit_Framework_TestCase
16{ 16{
@@ -38,7 +38,8 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
38 ]); 38 ]);
39 39
40 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage); 40 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage);
41 $entry = $proxy->updateEntry(new Entry(new User()), 'http://user@:80'); 41 $entry = new Entry(new User());
42 $proxy->updateEntry($entry, 'http://user@:80');
42 43
43 $this->assertEquals('http://user@:80', $entry->getUrl()); 44 $this->assertEquals('http://user@:80', $entry->getUrl());
44 $this->assertEmpty($entry->getTitle()); 45 $this->assertEmpty($entry->getTitle());
@@ -72,7 +73,8 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
72 ]); 73 ]);
73 74
74 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage); 75 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage);
75 $entry = $proxy->updateEntry(new Entry(new User()), 'http://0.0.0.0'); 76 $entry = new Entry(new User());
77 $proxy->updateEntry($entry, 'http://0.0.0.0');
76 78
77 $this->assertEquals('http://0.0.0.0', $entry->getUrl()); 79 $this->assertEquals('http://0.0.0.0', $entry->getUrl());
78 $this->assertEmpty($entry->getTitle()); 80 $this->assertEmpty($entry->getTitle());
@@ -111,7 +113,8 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
111 ]); 113 ]);
112 114
113 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage); 115 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage);
114 $entry = $proxy->updateEntry(new Entry(new User()), 'http://domain.io'); 116 $entry = new Entry(new User());
117 $proxy->updateEntry($entry, 'http://domain.io');
115 118
116 $this->assertEquals('http://domain.io', $entry->getUrl()); 119 $this->assertEquals('http://domain.io', $entry->getUrl());
117 $this->assertEquals('my title', $entry->getTitle()); 120 $this->assertEquals('my title', $entry->getTitle());
@@ -152,7 +155,8 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
152 ]); 155 ]);
153 156
154 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage); 157 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage);
155 $entry = $proxy->updateEntry(new Entry(new User()), 'http://0.0.0.0'); 158 $entry = new Entry(new User());
159 $proxy->updateEntry($entry, 'http://0.0.0.0');
156 160
157 $this->assertEquals('http://1.1.1.1', $entry->getUrl()); 161 $this->assertEquals('http://1.1.1.1', $entry->getUrl());
158 $this->assertEquals('this is my title', $entry->getTitle()); 162 $this->assertEquals('this is my title', $entry->getTitle());
@@ -193,7 +197,8 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
193 ]); 197 ]);
194 198
195 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage); 199 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage);
196 $entry = $proxy->updateEntry(new Entry(new User()), 'http://0.0.0.0'); 200 $entry = new Entry(new User());
201 $proxy->updateEntry($entry, 'http://0.0.0.0');
197 202
198 $this->assertEquals('http://1.1.1.1', $entry->getUrl()); 203 $this->assertEquals('http://1.1.1.1', $entry->getUrl());
199 $this->assertEquals('this is my title', $entry->getTitle()); 204 $this->assertEquals('this is my title', $entry->getTitle());
@@ -213,8 +218,9 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
213 ->method('tag'); 218 ->method('tag');
214 219
215 $proxy = new ContentProxy((new Graby()), $tagger, $this->getLogger(), $this->fetchingErrorMessage); 220 $proxy = new ContentProxy((new Graby()), $tagger, $this->getLogger(), $this->fetchingErrorMessage);
216 $entry = $proxy->updateEntry( 221 $entry = new Entry(new User());
217 new Entry(new User()), 222 $proxy->updateEntry(
223 $entry,
218 'http://0.0.0.0', 224 'http://0.0.0.0',
219 [ 225 [
220 'html' => str_repeat('this is my content', 325), 226 'html' => str_repeat('this is my content', 325),
@@ -250,10 +256,14 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
250 $tagger->expects($this->once()) 256 $tagger->expects($this->once())
251 ->method('tag'); 257 ->method('tag');
252 258
253 $proxy = new ContentProxy((new Graby()), $tagger, $this->getLogger(), $this->fetchingErrorMessage); 259 $logHandler = new TestHandler();
254 $entry = $proxy->updateEntry( 260 $logger = new Logger('test', [$logHandler]);
255 new Entry(new User()), 261
256 'http://0.0.0.0', 262 $proxy = new ContentProxy((new Graby()), $tagger, $logger, $this->fetchingErrorMessage);
263 $entry = new Entry(new User());
264 $proxy->updateEntry(
265 $entry,
266 'http://1.1.1.1',
257 [ 267 [
258 'html' => str_repeat('this is my content', 325), 268 'html' => str_repeat('this is my content', 325),
259 'title' => 'this is my title', 269 'title' => 'this is my title',
@@ -285,9 +295,10 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
285 $logger->pushHandler($handler); 295 $logger->pushHandler($handler);
286 296
287 $proxy = new ContentProxy((new Graby()), $tagger, $logger, $this->fetchingErrorMessage); 297 $proxy = new ContentProxy((new Graby()), $tagger, $logger, $this->fetchingErrorMessage);
288 $entry = $proxy->updateEntry( 298 $entry = new Entry(new User());
289 new Entry(new User()), 299 $proxy->updateEntry(
290 'http://0.0.0.0', 300 $entry,
301 'http://1.1.1.1',
291 [ 302 [
292 'html' => str_repeat('this is my content', 325), 303 'html' => str_repeat('this is my content', 325),
293 'title' => 'this is my title', 304 'title' => 'this is my title',
@@ -315,24 +326,24 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
315 326
316 public function testTaggerThrowException() 327 public function testTaggerThrowException()
317 { 328 {
318 $graby = $this->getMockBuilder('Graby\Graby')
319 ->disableOriginalConstructor()
320 ->getMock();
321
322 $tagger = $this->getTaggerMock(); 329 $tagger = $this->getTaggerMock();
323 $tagger->expects($this->once()) 330 $tagger->expects($this->once())
324 ->method('tag') 331 ->method('tag')
325 ->will($this->throwException(new \Exception())); 332 ->will($this->throwException(new \Exception()));
326 333
327 $proxy = new ContentProxy($graby, $tagger, $this->getLogger(), $this->fetchingErrorMessage); 334 $proxy = new ContentProxy((new Graby()), $tagger, $this->getLogger(), $this->fetchingErrorMessage);
328 335 $entry = new Entry(new User());
329 $entry = $proxy->updateEntry(new Entry(new User()), 'http://0.0.0.0', [ 336 $proxy->updateEntry(
330 'html' => str_repeat('this is my content', 325), 337 $entry,
331 'title' => 'this is my title', 338 'http://1.1.1.1',
332 'url' => 'http://1.1.1.1', 339 [
333 'content_type' => 'text/html', 340 'html' => str_repeat('this is my content', 325),
334 'language' => 'fr', 341 'title' => 'this is my title',
335 ]); 342 'url' => 'http://1.1.1.1',
343 'content_type' => 'text/html',
344 'language' => 'fr',
345 ]
346 );
336 347
337 $this->assertCount(0, $entry->getTags()); 348 $this->assertCount(0, $entry->getTags());
338 } 349 }
@@ -361,8 +372,9 @@ class ContentProxyTest extends \PHPUnit_Framework_TestCase
361 ->method('tag'); 372 ->method('tag');
362 373
363 $proxy = new ContentProxy((new Graby()), $tagger, $this->getLogger(), $this->fetchingErrorMessage); 374 $proxy = new ContentProxy((new Graby()), $tagger, $this->getLogger(), $this->fetchingErrorMessage);
364 $entry = $proxy->updateEntry( 375 $entry = new Entry(new User());
365 new Entry(new User()), 376 $proxy->updateEntry(
377 $entry,
366 'http://1.1.1.1', 378 'http://1.1.1.1',
367 [ 379 [
368 'html' => $html, 380 'html' => $html,
diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
index 7cbef637..834b7ef5 100644
--- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
+++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php
@@ -19,6 +19,8 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
19 protected $contentProxy; 19 protected $contentProxy;
20 protected $tagsAssigner; 20 protected $tagsAssigner;
21 protected $uow; 21 protected $uow;
22 protected $fetchingErrorMessageTitle = 'No title found';
23 protected $fetchingErrorMessage = 'wallabag can\'t retrieve contents for this article. Please <a href="http://doc.wallabag.org/en/master/user/errors_during_fetching.html#how-can-i-help-to-fix-that">troubleshoot this issue</a>.';
22 24
23 private function getWallabagV1Import($unsetUser = false, $dispatched = 0) 25 private function getWallabagV1Import($unsetUser = false, $dispatched = 0)
24 { 26 {
@@ -58,7 +60,14 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase
58 ->expects($this->exactly($dispatched)) 60 ->expects($this->exactly($dispatched))
59 ->method('dispatch'); 61 ->method('dispatch');
60 62
61 $wallabag = new WallabagV1Import($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher); 63 $wallabag = new WallabagV1Import(
64 $this->em,
65 $this->contentProxy,
66 $this->tagsAssigner,
67 $dispatcher,
68 $this->fetchingErrorMessageTitle,
69 $this->fetchingErrorMessage
70 );
62 71
63 $this->logHandler = new TestHandler(); 72 $this->logHandler = new TestHandler();
64 $logger = new Logger('test', [$this->logHandler]); 73 $logger = new Logger('test', [$this->logHandler]);