From: Jeremy Benoist Date: Thu, 3 Nov 2016 15:22:07 +0000 (+0100) Subject: Merge pull request #2180 from wallabag/download-pictures X-Git-Tag: 2.2.0~3^2~79 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=da4136557963018287cae61226e9006c3c741747;hp=816c0940d142f8ec69bdd3f302c9751033cce7e2;p=github%2Fwallabag%2Fwallabag.git Merge pull request #2180 from wallabag/download-pictures Download pictures --- diff --git a/.gitignore b/.gitignore index 32b0fbbb..84fb95d7 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ web/uploads/ !web/bundles web/bundles/* !web/bundles/wallabagcore +/web/assets/images/* +!web/assets/images/.gitkeep # Build /app/build @@ -34,7 +36,6 @@ web/bundles/* /composer.phar # Data for wallabag -data/assets/* data/db/wallabag*.sqlite # Docker container logs and data diff --git a/app/DoctrineMigrations/Version20161031132655.php b/app/DoctrineMigrations/Version20161031132655.php new file mode 100644 index 00000000..c7364428 --- /dev/null +++ b/app/DoctrineMigrations/Version20161031132655.php @@ -0,0 +1,44 @@ +container = $container; + } + + private function getTable($tableName) + { + return $this->container->getParameter('database_table_prefix') . $tableName; + } + + /** + * @param Schema $schema + */ + public function up(Schema $schema) + { + $this->addSql("INSERT INTO \"".$this->getTable('craue_config_setting')."\" (name, value, section) VALUES ('download_images_enabled', 0, 'misc')"); + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.'); + + $this->addSql("DELETE FROM \"".$this->getTable('craue_config_setting')."\" WHERE name = 'download_images_enabled';"); + } +} diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml index 3e11d675..7c323783 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml @@ -29,3 +29,4 @@ piwik_enabled: Aktiver Piwik demo_mode_enabled: "Aktiver demo-indstilling? (anvendes kun til wallabags offentlige demo)" demo_mode_username: "Demobruger" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml index c74b5c1f..438eb74a 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml @@ -29,3 +29,4 @@ piwik_enabled: Piwik aktivieren demo_mode_enabled: "Test-Modus aktivieren? (nur für die öffentliche wallabag-Demo genutzt)" demo_mode_username: "Test-Benutzer" share_public: Erlaube eine öffentliche URL für Einträge +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml index 77c09db4..c2f2b3fb 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml @@ -29,3 +29,4 @@ piwik_enabled: Enable Piwik demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" demo_mode_username: "Demo user" share_public: Allow public url for entries +download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml index baa83849..76feea50 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml @@ -29,3 +29,4 @@ piwik_enabled: Activar Piwik demo_mode_enabled: "Activar modo demo (sólo usado para la demo de wallabag)" demo_mode_username: "Nombre de usuario demo" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml index b394977e..30df0086 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml @@ -29,3 +29,4 @@ modify_settings: "اعمال" # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" # demo_mode_username: "Demo user" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml index 31a80880..a60341b3 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml @@ -29,3 +29,4 @@ piwik_enabled: Activer Piwik demo_mode_enabled: "Activer le mode démo ? (utiliser uniquement pour la démo publique de wallabag)" demo_mode_username: "Utilisateur de la démo" share_public: Autoriser une URL publique pour les articles +download_images_enabled: Télécharger les images en local diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml index ba038556..3ad5f7d0 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml @@ -29,3 +29,4 @@ piwik_enabled: Abilita Piwik demo_mode_enabled: "Abilita modalità demo ? (usato solo per la demo pubblica di wallabag)" demo_mode_username: "Utente Demo" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml index 55249e33..fd83b437 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml @@ -29,3 +29,4 @@ piwik_enabled: Activar Piwik demo_mode_enabled: "Activar lo mode demostracion ? (utilizar solament per la demostracion publica de wallabag)" demo_mode_username: "Utilizaire de la demostracion" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml index 42cc5b52..3a63eebb 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml @@ -29,3 +29,4 @@ piwik_enabled: Włacz Piwik demo_mode_enabled: "Włacz tryb demo? (używany wyłącznie dla publicznej demonstracji Wallabag)" demo_mode_username: "Użytkownik Demonstracyjny" share_public: Zezwalaj na publiczny adres url dla wpisow +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml index 8e72b955..4fb42e98 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml @@ -29,3 +29,4 @@ modify_settings: "aplică" # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" # demo_mode_username: "Demo user" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml index 55f70843..ebfadf29 100644 --- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml +++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml @@ -29,3 +29,4 @@ # demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)" # demo_mode_username: "Demo user" # share_public: Allow public url for entries +# download_images_enabled: Download images locally diff --git a/app/config/services.yml b/app/config/services.yml index a57ef0f3..9a1ce80b 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -32,13 +32,13 @@ services: - { name: twig.extension } wallabag.locale_listener: - class: Wallabag\CoreBundle\EventListener\LocaleListener + class: Wallabag\CoreBundle\Event\Listener\LocaleListener arguments: ["%kernel.default_locale%"] tags: - { name: kernel.event_subscriber } wallabag.user_locale_listener: - class: Wallabag\CoreBundle\EventListener\UserLocaleListener + class: Wallabag\CoreBundle\Event\Listener\UserLocaleListener arguments: ["@session"] tags: - { name: kernel.event_listener, event: security.interactive_login, method: onInteractiveLogin } diff --git a/composer.json b/composer.json index ebc0a7dc..e7b30fa1 100644 --- a/composer.json +++ b/composer.json @@ -82,7 +82,8 @@ "white-october/pagerfanta-bundle": "^1.0", "php-amqplib/rabbitmq-bundle": "^1.8", "predis/predis": "^1.0", - "javibravo/simpleue": "^1.0" + "javibravo/simpleue": "^1.0", + "symfony/dom-crawler": "^3.1" }, "require-dev": { "doctrine/doctrine-fixtures-bundle": "~2.2", diff --git a/src/Wallabag/ApiBundle/Controller/WallabagRestController.php b/src/Wallabag/ApiBundle/Controller/WallabagRestController.php index a73d44ca..50652b77 100644 --- a/src/Wallabag/ApiBundle/Controller/WallabagRestController.php +++ b/src/Wallabag/ApiBundle/Controller/WallabagRestController.php @@ -14,6 +14,8 @@ use Symfony\Component\Security\Core\Exception\AccessDeniedException; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\AnnotationBundle\Entity\Annotation; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; class WallabagRestController extends FOSRestController { @@ -233,9 +235,11 @@ class WallabagRestController extends FOSRestController $em = $this->getDoctrine()->getManager(); $em->persist($entry); - $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + $json = $this->get('serializer')->serialize($entry, 'json'); return (new JsonResponse())->setJson($json); @@ -308,6 +312,9 @@ class WallabagRestController extends FOSRestController $this->validateAuthentication(); $this->validateUserAccess($entry->getUser()->getId()); + // entry deleted, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry)); + $em = $this->getDoctrine()->getManager(); $em->remove($entry); $em->flush(); diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 277f8524..9fe90357 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -370,7 +370,7 @@ class InstallCommand extends ContainerAwareCommand ], [ 'name' => 'wallabag_url', - 'value' => 'http://v2.wallabag.org', + 'value' => '', 'section' => 'misc', ], [ @@ -398,6 +398,11 @@ class InstallCommand extends ContainerAwareCommand 'value' => 'wallabag', 'section' => 'misc', ], + [ + 'name' => 'download_images_enabled', + 'value' => '0', + 'section' => 'misc', + ], ]; foreach ($settings as $setting) { diff --git a/src/Wallabag/CoreBundle/Controller/EntryController.php b/src/Wallabag/CoreBundle/Controller/EntryController.php index 97bb3d12..3f4eb17d 100644 --- a/src/Wallabag/CoreBundle/Controller/EntryController.php +++ b/src/Wallabag/CoreBundle/Controller/EntryController.php @@ -13,6 +13,8 @@ use Wallabag\CoreBundle\Form\Type\EntryFilterType; use Wallabag\CoreBundle\Form\Type\EditEntryType; use Wallabag\CoreBundle\Form\Type\NewEntryType; use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; class EntryController extends Controller { @@ -81,6 +83,9 @@ class EntryController extends Controller $em->persist($entry); $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + return $this->redirect($this->generateUrl('homepage')); } @@ -107,6 +112,9 @@ class EntryController extends Controller $em = $this->getDoctrine()->getManager(); $em->persist($entry); $em->flush(); + + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); } return $this->redirect($this->generateUrl('homepage')); @@ -343,6 +351,9 @@ class EntryController extends Controller $em->persist($entry); $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()])); } @@ -431,6 +442,9 @@ class EntryController extends Controller UrlGeneratorInterface::ABSOLUTE_PATH ); + // entry deleted, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry)); + $em = $this->getDoctrine()->getManager(); $em->remove($entry); $em->flush(); diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index a5e1be65..d0085660 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -140,6 +140,11 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface 'value' => 'wallabag', 'section' => 'misc', ], + [ + 'name' => 'download_images_enabled', + 'value' => '0', + 'section' => 'misc', + ], ]; foreach ($settings as $setting) { @@ -158,6 +163,6 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface */ public function getOrder() { - return 50; + return 29; } } diff --git a/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php b/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php new file mode 100644 index 00000000..e9061d04 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php @@ -0,0 +1,26 @@ +entry = $entry; + } + + public function getEntry() + { + return $this->entry; + } +} diff --git a/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php b/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php new file mode 100644 index 00000000..5fdb5221 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php @@ -0,0 +1,26 @@ +entry = $entry; + } + + public function getEntry() + { + return $this->entry; + } +} diff --git a/src/Wallabag/CoreBundle/EventListener/LocaleListener.php b/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php similarity index 96% rename from src/Wallabag/CoreBundle/EventListener/LocaleListener.php rename to src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php index a1c7e5ab..b435d99e 100644 --- a/src/Wallabag/CoreBundle/EventListener/LocaleListener.php +++ b/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php @@ -1,6 +1,6 @@ em = $em; + $this->downloadImages = $downloadImages; + $this->enabled = $enabled; + $this->logger = $logger; + } + + public static function getSubscribedEvents() + { + return [ + EntrySavedEvent::NAME => 'onEntrySaved', + EntryDeletedEvent::NAME => 'onEntryDeleted', + ]; + } + + /** + * Download images and updated the data into the entry. + * + * @param EntrySavedEvent $event + */ + public function onEntrySaved(EntrySavedEvent $event) + { + if (!$this->enabled) { + $this->logger->debug('DownloadImagesSubscriber: disabled.'); + + return; + } + + $entry = $event->getEntry(); + + $html = $this->downloadImages($entry); + if (false !== $html) { + $this->logger->debug('DownloadImagesSubscriber: updated html.'); + + $entry->setContent($html); + } + + // update preview picture + $previewPicture = $this->downloadPreviewImage($entry); + if (false !== $previewPicture) { + $this->logger->debug('DownloadImagesSubscriber: update preview picture.'); + + $entry->setPreviewPicture($previewPicture); + } + + $this->em->persist($entry); + $this->em->flush(); + } + + /** + * Remove images related to the entry. + * + * @param EntryDeletedEvent $event + */ + public function onEntryDeleted(EntryDeletedEvent $event) + { + if (!$this->enabled) { + $this->logger->debug('DownloadImagesSubscriber: disabled.'); + + return; + } + + $this->downloadImages->removeImages($event->getEntry()->getId()); + } + + /** + * Download all images from the html. + * + * @todo If we want to add async download, it should be done in that method + * + * @param Entry $entry + * + * @return string|false False in case of async + */ + private function downloadImages(Entry $entry) + { + return $this->downloadImages->processHtml( + $entry->getId(), + $entry->getContent(), + $entry->getUrl() + ); + } + + /** + * Download the preview picture. + * + * @todo If we want to add async download, it should be done in that method + * + * @param Entry $entry + * + * @return string|false False in case of async + */ + private function downloadPreviewImage(Entry $entry) + { + return $this->downloadImages->processSingleImage( + $entry->getId(), + $entry->getPreviewPicture(), + $entry->getUrl() + ); + } +} diff --git a/src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php similarity index 97% rename from src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php rename to src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php index f7210bd3..3b4c4cf9 100644 --- a/src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php @@ -1,6 +1,6 @@ graby = $graby; $this->tagger = $tagger; @@ -66,6 +66,7 @@ class ContentProxy $entry->setUrl($content['url'] ?: $url); $entry->setTitle($title); $entry->setContent($html); + $entry->setLanguage($content['language']); $entry->setMimetype($content['content_type']); $entry->setReadingTime(Utils::getReadingTime($html)); diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php new file mode 100644 index 00000000..c5298236 --- /dev/null +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -0,0 +1,233 @@ +client = $client; + $this->baseFolder = $baseFolder; + $this->wallabagUrl = rtrim($wallabagUrl, '/'); + $this->logger = $logger; + $this->mimeGuesser = new MimeTypeExtensionGuesser(); + + $this->setFolder(); + } + + /** + * Setup base folder where all images are going to be saved. + */ + private function setFolder() + { + // if folder doesn't exist, attempt to create one and store the folder name in property $folder + if (!file_exists($this->baseFolder)) { + mkdir($this->baseFolder, 0777, true); + } + } + + /** + * Process the html and extract image from it, save them to local and return the updated html. + * + * @param int $entryId ID of the entry + * @param string $html + * @param string $url Used as a base path for relative image and folder + * + * @return string + */ + public function processHtml($entryId, $html, $url) + { + $crawler = new Crawler($html); + $result = $crawler + ->filterXpath('//img') + ->extract(array('src')); + + $relativePath = $this->getRelativePath($entryId); + + // download and save the image to the folder + foreach ($result as $image) { + $imagePath = $this->processSingleImage($entryId, $image, $url, $relativePath); + + if (false === $imagePath) { + continue; + } + + $html = str_replace($image, $imagePath, $html); + } + + return $html; + } + + /** + * Process a single image: + * - retrieve it + * - re-saved it (for security reason) + * - return the new local path. + * + * @param int $entryId ID of the entry + * @param string $imagePath Path to the image to retrieve + * @param string $url Url from where the image were found + * @param string $relativePath Relative local path to saved the image + * + * @return string Relative url to access the image from the web + */ + public function processSingleImage($entryId, $imagePath, $url, $relativePath = null) + { + if (null === $relativePath) { + $relativePath = $this->getRelativePath($entryId); + } + + $this->logger->debug('DownloadImages: working on image: '.$imagePath); + + $folderPath = $this->baseFolder.'/'.$relativePath; + + // build image path + $absolutePath = $this->getAbsoluteLink($url, $imagePath); + if (false === $absolutePath) { + $this->logger->error('DownloadImages: Can not determine the absolute path for that image, skipping.'); + + return false; + } + + try { + $res = $this->client->get($absolutePath); + } catch (\Exception $e) { + $this->logger->error('DownloadImages: Can not retrieve image, skipping.', ['exception' => $e]); + + return false; + } + + $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); + $this->logger->debug('DownloadImages: Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); + if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { + $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping '.$imagePath); + + return false; + } + $hashImage = hash('crc32', $absolutePath); + $localPath = $folderPath.'/'.$hashImage.'.'.$ext; + + try { + $im = imagecreatefromstring($res->getBody()); + } catch (\Exception $e) { + $im = false; + } + + if (false === $im) { + $this->logger->error('DownloadImages: Error while regenerating image', ['path' => $localPath]); + + return false; + } + + switch ($ext) { + case 'gif': + $result = imagegif($im, $localPath); + $this->logger->debug('DownloadImages: Re-creating gif'); + break; + case 'jpeg': + case 'jpg': + $result = imagejpeg($im, $localPath, self::REGENERATE_PICTURES_QUALITY); + $this->logger->debug('DownloadImages: Re-creating jpg'); + break; + case 'png': + $result = imagepng($im, $localPath, ceil(self::REGENERATE_PICTURES_QUALITY / 100 * 9)); + $this->logger->debug('DownloadImages: Re-creating png'); + } + + imagedestroy($im); + + return $this->wallabagUrl.'/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; + } + + /** + * Remove all images for the given entry id. + * + * @param int $entryId ID of the entry + */ + public function removeImages($entryId) + { + $relativePath = $this->getRelativePath($entryId); + $folderPath = $this->baseFolder.'/'.$relativePath; + + $finder = new Finder(); + $finder + ->files() + ->ignoreDotFiles(true) + ->in($folderPath); + + foreach ($finder as $file) { + @unlink($file->getRealPath()); + } + + @rmdir($folderPath); + } + + /** + * Generate the folder where we are going to save images based on the entry url. + * + * @param int $entryId ID of the entry + * + * @return string + */ + private function getRelativePath($entryId) + { + $hashId = hash('crc32', $entryId); + $relativePath = $hashId[0].'/'.$hashId[1].'/'.$hashId; + $folderPath = $this->baseFolder.'/'.$relativePath; + + if (!file_exists($folderPath)) { + mkdir($folderPath, 0777, true); + } + + $this->logger->debug('DownloadImages: Folder used for that Entry id', ['folder' => $folderPath, 'entryId' => $entryId]); + + return $relativePath; + } + + /** + * Make an $url absolute based on the $base. + * + * @see Graby->makeAbsoluteStr + * + * @param string $base Base url + * @param string $url Url to make it absolute + * + * @return false|string + */ + private function getAbsoluteLink($base, $url) + { + if (preg_match('!^https?://!i', $url)) { + // already absolute + return $url; + } + + $base = new \SimplePie_IRI($base); + + // remove '//' in URL path (causes URLs not to resolve properly) + if (isset($base->ipath)) { + $base->ipath = preg_replace('!//+!', '/', $base->ipath); + } + + if ($absolute = \SimplePie_IRI::absolutize($base, $url)) { + return $absolute->get_uri(); + } + + $this->logger->error('DownloadImages: Can not make an absolute link', ['base' => $base, 'url' => $url]); + + return false; + } +} diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index cc5f9e9a..56d776ad 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -30,7 +30,7 @@ services: - "@doctrine" wallabag_core.subscriber.table_prefix: - class: Wallabag\CoreBundle\Subscriber\TablePrefixSubscriber + class: Wallabag\CoreBundle\Event\Subscriber\TablePrefixSubscriber arguments: - "%database_table_prefix%" tags: @@ -131,8 +131,29 @@ services: - '%kernel.debug%' wallabag_core.subscriber.sqlite_cascade_delete: - class: Wallabag\CoreBundle\Subscriber\SQLiteCascadeDeleteSubscriber + class: Wallabag\CoreBundle\Event\Subscriber\SQLiteCascadeDeleteSubscriber arguments: - "@doctrine" tags: - { name: doctrine.event_subscriber } + + wallabag_core.subscriber.download_images: + class: Wallabag\CoreBundle\Event\Subscriber\DownloadImagesSubscriber + arguments: + - "@doctrine.orm.default_entity_manager" + - "@wallabag_core.entry.download_images" + - '@=service(''craue_config'').get(''download_images_enabled'')' + - "@logger" + tags: + - { name: kernel.event_subscriber } + + wallabag_core.entry.download_images: + class: Wallabag\CoreBundle\Helper\DownloadImages + arguments: + - "@wallabag_core.entry.download_images.client" + - "%kernel.root_dir%/../web/assets/images" + - '@=service(''craue_config'').get(''wallabag_url'')' + - "@logger" + + wallabag_core.entry.download_images.client: + class: GuzzleHttp\Client diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index 9b0eecce..21c26079 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -368,6 +368,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." # firefox: # page_title: 'Import > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -505,3 +506,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index a156c54a..ff70cbee 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -368,6 +368,7 @@ import: how_to: 'Bitte wähle deinen Readability Export aus und klicke den unteren Button für das Hochladen und Importieren dessen.' worker: enabled: "Der Import erfolgt asynchron. Sobald der Import gestartet ist, wird diese Aufgabe extern abgearbeitet. Der aktuelle Service dafür ist:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Aus Firefox importieren' description: "Dieser Import wird all deine Lesezeichen aus Firefox importieren. Gehe zu deinen Lesezeichen (Strg+Shift+O), dann auf \"Importen und Sichern\", wähle \"Sichern…\". Du erhälst eine .json Datei." @@ -505,3 +506,8 @@ flashes: notice: client_created: 'Neuer Client erstellt.' client_deleted: 'Client gelöscht' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 049959a0..36382b6f 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -368,6 +368,7 @@ import: how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Import > Firefox' description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -471,6 +472,7 @@ flashes: rss_updated: 'RSS information updated' tagging_rules_updated: 'Tagging rules updated' tagging_rules_deleted: 'Tagging rule deleted' + # user_added: 'User "%username%" added' rss_token_updated: 'RSS token updated' annotations_reset: Annotations reset tags_reset: Tags reset diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index 79c13ff0..2c80fe8f 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -368,6 +368,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -505,3 +506,8 @@ flashes: notice: client_created: 'Nuevo cliente creado.' client_deleted: 'Cliente suprimido' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index e81513aa..6b6211d6 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -285,6 +285,7 @@ quickstart: paragraph_2: 'ادامه دهید!' configure: title: 'برنامه را تنظیم کنید' + # description: 'In order to have an application which suits you, have a look into the configuration of wallabag.' language: 'زبان و نمای برنامه را تغییر دهید' rss: 'خوراک آر-اس-اس را فعال کنید' tagging_rules: 'قانون‌های برچسب‌گذاری خودکار مقاله‌هایتان را تعریف کنید' @@ -367,6 +368,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'درون‌ریزی > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +506,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index ac7c112d..74d59e1a 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -368,6 +368,7 @@ import: how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer." worker: enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :" + download_images_warning: "Vous avez configuré le téléchagement des images pour vos articles. Combiné à l'import classique, cette opération peut être très très longue (voire échouer). Nous vous conseillons vivement d'activer les imports asynchrones." firefox: page_title: 'Import > Firefox' description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox. Ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json.

" @@ -505,3 +506,8 @@ flashes: notice: client_created: 'Nouveau client %name% créé' client_deleted: 'Client %name% supprimé' + user: + notice: + added: 'Utilisateur "%username%" ajouté' + updated: 'Utilisateur "%username%" mis à jour' + deleted: 'Utilisateur "%username%" supprimé' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 454429f6..a448b602 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -368,6 +368,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importa da > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -505,3 +506,8 @@ flashes: notice: client_created: 'Nuovo client creato.' client_deleted: 'Client eliminato' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index dadb6b03..a61f7cdd 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -368,6 +368,7 @@ import: how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." worker: enabled: "L'importacion se fa de manièra asincròna. Un còp l'importacion lançada, una aisina externa s'ocuparà dels messatges un per un. Lo servici actual es : " + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' description: "Aquesta aisina importarà totas vòstres favorits de Firefox. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -505,3 +506,8 @@ flashes: notice: client_created: 'Novèl client creat' client_deleted: 'Client suprimit' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index 200907c9..a7387b79 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -368,6 +368,7 @@ import: how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' worker: enabled: "Import jest wykonywany asynchronicznie. Od momentu rozpoczęcia importu, zewnętrzna usługa może zajmować się na raz tylko jednym zadaniem. Bieżącą usługą jest:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Import > Firefox' description: "Ten importer zaimportuje wszystkie twoje zakładki z Firefoksa. Idź do twoich zakładek (Ctrl+Shift+O), następnie w \"Import i kopie zapasowe\", wybierz \"Utwórz kopię zapasową...\". Uzyskasz plik .json." @@ -505,3 +506,8 @@ flashes: notice: client_created: 'Nowy klient utworzony.' client_deleted: 'Klient usunięty' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 255ee839..070abe27 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -368,6 +368,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." # firefox: # page_title: 'Import > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -505,3 +506,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index f3696a55..7679b32a 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -109,6 +109,7 @@ config: # if_label: 'if' # then_tag_as_label: 'then tag as' # delete_rule_label: 'delete' + # edit_rule_label: 'edit' rule_label: 'Kural' tags_label: 'Etiketler' faq: @@ -367,6 +368,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'İçe Aktar > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +506,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 1df38295..2f7a906e 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -14,10 +14,10 @@ class ImportCommand extends ContainerAwareCommand { $this ->setName('wallabag:import') - ->setDescription('Import entries from a JSON export from a wallabag v1 instance') + ->setDescription('Import entries from a JSON export') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, readability, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -42,29 +42,35 @@ class ImportCommand extends ContainerAwareCommand switch ($input->getOption('importer')) { case 'v2': - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + $import = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); break; case 'firefox': - $wallabag = $this->getContainer()->get('wallabag_import.firefox.import'); + $import = $this->getContainer()->get('wallabag_import.firefox.import'); break; case 'chrome': - $wallabag = $this->getContainer()->get('wallabag_import.chrome.import'); + $import = $this->getContainer()->get('wallabag_import.chrome.import'); + break; + case 'readability': + $import = $this->getContainer()->get('wallabag_import.readability.import'); + break; + case 'instapaper': + $import = $this->getContainer()->get('wallabag_import.instapaper.import'); break; case 'v1': default: - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); + $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); break; } - $wallabag->setMarkAsRead($input->getOption('markAsRead')); - $wallabag->setUser($user); + $import->setMarkAsRead($input->getOption('markAsRead')); + $import->setUser($user); - $res = $wallabag + $res = $import ->setFilepath($input->getArgument('filepath')) ->import(); if (true === $res) { - $summary = $wallabag->getSummary(); + $summary = $import->getSummary(); $output->writeln(''.$summary['imported'].' imported'); $output->writeln(''.$summary['skipped'].' already saved'); } diff --git a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php index b893ea29..aa7ff914 100644 --- a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php +++ b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php @@ -9,6 +9,8 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractConsumer { @@ -17,11 +19,12 @@ abstract class AbstractConsumer protected $import; protected $logger; - public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null) + public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, EventDispatcherInterface $eventDispatcher, LoggerInterface $logger = null) { $this->em = $em; $this->userRepository = $userRepository; $this->import = $import; + $this->eventDispatcher = $eventDispatcher; $this->logger = $logger ?: new NullLogger(); } @@ -59,6 +62,9 @@ abstract class AbstractConsumer try { $this->em->flush(); + // entry saved, dispatch event about it! + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 764b390a..1d4a6e27 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractImport implements ImportInterface { protected $em; protected $logger; protected $contentProxy; + protected $eventDispatcher; protected $producer; protected $user; protected $markAsRead; @@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface protected $importedEntries = 0; protected $queuedEntries = 0; - public function __construct(EntityManager $em, ContentProxy $contentProxy) + public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher) { $this->em = $em; $this->logger = new NullLogger(); $this->contentProxy = $contentProxy; + $this->eventDispatcher = $eventDispatcher; } public function setLogger(LoggerInterface $logger) @@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ($this->markAsRead) { @@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface continue; } + // store each entry to be flushed so we can trigger the entry.saved event for each of them + // entry.saved needs the entry to be persisted in db because it needs it id to generate + // images (at least) + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); @@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 2ca1683b..8bf7d92e 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class BrowserImport extends AbstractImport { @@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ((array) $importedEntry !== $importedEntry) { @@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport continue; } + // @see AbstractImport + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; } ++$i; } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 327e2500..33093480 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,8 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; @@ -16,13 +14,6 @@ class PocketImport extends AbstractImport const NB_ELEMENTS = 5000; - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->contentProxy = $contentProxy; - $this->logger = new NullLogger(); - } - /** * Only used for test purpose. * diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index 70b8a0d4..a5af5282 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -6,6 +6,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.readability: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -13,6 +14,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.instapaper: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -20,6 +22,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.instapaper.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -27,6 +30,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.wallabag_v2: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -34,6 +38,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.firefox: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -41,6 +46,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.firefox.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.chrome: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -48,4 +54,5 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.chrome.import" + - "@event_dispatcher" - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 0a81e1b5..5ced4c83 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -18,6 +18,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" + - "@event_dispatcher" - "@logger" # instapaper @@ -38,6 +39,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.instapaper.import" + - "@event_dispatcher" - "@logger" # pocket @@ -58,6 +60,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" + - "@event_dispatcher" - "@logger" # wallabag v1 @@ -78,6 +81,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" + - "@event_dispatcher" - "@logger" # wallabag v2 @@ -98,6 +102,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" + - "@event_dispatcher" - "@logger" # firefox @@ -118,6 +123,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.firefox.import" + - "@event_dispatcher" - "@logger" # chrome @@ -138,4 +144,5 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.chrome.import" + - "@event_dispatcher" - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 89adc71b..64822963 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -20,7 +20,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" - - "@craue_config" + - "@event_dispatcher" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] @@ -32,6 +32,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -42,6 +43,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -52,6 +54,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -62,6 +65,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -72,6 +76,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -81,6 +86,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig index ead828c6..93b08540 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} +
{{ import.description|trans|raw }}

{{ 'import.chrome.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig index f975da3f..ced3f008 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} +
{{ import.description|trans|raw }}

{{ 'import.firefox.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig similarity index 55% rename from src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig rename to src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig index 2390a41f..48bbcfe7 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig @@ -1,8 +1,15 @@ {% set redis = craue_setting('import_with_redis') %} {% set rabbit = craue_setting('import_with_rabbitmq') %} +{% set downloadImages = craue_setting('download_images_enabled') %} {% if redis or rabbit %}
{{ 'import.worker.enabled'|trans }} {% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}
{% endif %} + +{% if not redis and not rabbit and downloadImages %} +
+ {{ 'import.worker.download_images_warning'|trans|raw }} +
+{% endif %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index 6ea5e0f4..b1ec40a6 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} + {{ 'import.page_description'|trans }}
    {% for import in imports %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig index 5789361f..28165d19 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig index 6195fa07..536e3d1a 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %} {% if not has_consumer_key %}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig index 74653b0f..737b0adf 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig index 0b19bc34..974b2c73 100644 --- a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php b/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php index 05113650..4ab06dbf 100644 --- a/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php +++ b/tests/Wallabag/CoreBundle/Controller/EntryControllerTest.php @@ -836,4 +836,64 @@ class EntryControllerTest extends WallabagCoreTestCase $client->request('GET', '/share/'.$content->getUuid()); $this->assertEquals(404, $client->getResponse()->getStatusCode()); } + + public function testNewEntryWithDownloadImagesEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $url = 'http://www.20minutes.fr/montpellier/1952003-20161030-video-car-tombe-panne-rugbymen-perpignan-improvisent-melee-route'; + $client->getContainer()->get('craue_config')->set('download_images_enabled', 1); + + $crawler = $client->request('GET', '/new'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + + $form = $crawler->filter('form[name=entry]')->form(); + + $data = [ + 'entry[url]' => $url, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $em = $client->getContainer() + ->get('doctrine.orm.entity_manager'); + + $entry = $em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->getLoggedInUserId()); + + $this->assertInstanceOf('Wallabag\CoreBundle\Entity\Entry', $entry); + $this->assertEquals($url, $entry->getUrl()); + $this->assertContains('Perpignan', $entry->getTitle()); + $this->assertContains('/d9bc0fcd.jpeg', $entry->getContent()); + + $client->getContainer()->get('craue_config')->set('download_images_enabled', 0); + } + + /** + * @depends testNewEntryWithDownloadImagesEnabled + */ + public function testRemoveEntryWithDownloadImagesEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $url = 'http://www.20minutes.fr/montpellier/1952003-20161030-video-car-tombe-panne-rugbymen-perpignan-improvisent-melee-route'; + $client->getContainer()->get('craue_config')->set('download_images_enabled', 1); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($url, $this->getLoggedInUserId()); + + $client->request('GET', '/delete/'.$content->getId()); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $client->getContainer()->get('craue_config')->set('download_images_enabled', 0); + } } diff --git a/tests/Wallabag/CoreBundle/EventListener/LocaleListenerTest.php b/tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php similarity index 96% rename from tests/Wallabag/CoreBundle/EventListener/LocaleListenerTest.php rename to tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php index 078bb69a..84a54d3a 100644 --- a/tests/Wallabag/CoreBundle/EventListener/LocaleListenerTest.php +++ b/tests/Wallabag/CoreBundle/Event/Listener/LocaleListenerTest.php @@ -1,6 +1,6 @@ 'image/png'], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + + $res = $download->processHtml(123, '
    ', 'http://imgur.com/gallery/WxtWY'); + + $this->assertContains('http://wallabag.io/assets/images/9/b/9b0ead26/c638b4c2.png', $res); + } + + public function testProcessHtmlWithBadImage() + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => 'application/json'], Stream::factory('')), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processHtml(123, '
    ', 'http://imgur.com/gallery/WxtWY'); + + $this->assertContains('http://i.imgur.com/T9qgcHc.jpg', $res, 'Image were not replace because of content-type'); + } + + public function singleImage() + { + return [ + ['image/pjpeg', 'jpeg'], + ['image/jpeg', 'jpeg'], + ['image/png', 'png'], + ['image/gif', 'gif'], + ]; + } + + /** + * @dataProvider singleImage + */ + public function testProcessSingleImage($header, $extension) + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => $header], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, 'T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + + $this->assertContains('/assets/images/9/b/9b0ead26/ebe60399.'.$extension, $res); + } + + public function testProcessSingleImageWithBadUrl() + { + $client = new Client(); + + $mock = new Mock([ + new Response(404, []), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, 'T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + + $this->assertFalse($res, 'Image can not be found, so it will not be replaced'); + } + + public function testProcessSingleImageWithBadImage() + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => 'image/png'], Stream::factory('')), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, 'http://i.imgur.com/T9qgcHc.jpg', 'http://imgur.com/gallery/WxtWY'); + + $this->assertFalse($res, 'Image can not be loaded, so it will not be replaced'); + } + + public function testProcessSingleImageFailAbsolute() + { + $client = new Client(); + + $mock = new Mock([ + new Response(200, ['content-type' => 'image/png'], Stream::factory(file_get_contents(__DIR__.'/../fixtures/unnamed.png'))), + ]); + + $client->getEmitter()->attach($mock); + + $logHandler = new TestHandler(); + $logger = new Logger('test', array($logHandler)); + + $download = new DownloadImages($client, sys_get_temp_dir().'/wallabag_test', 'http://wallabag.io/', $logger); + $res = $download->processSingleImage(123, '/i.imgur.com/T9qgcHc.jpg', 'imgur.com/gallery/WxtWY'); + + $this->assertFalse($res, 'Absolute image can not be determined, so it will not be replaced'); + } +} diff --git a/tests/Wallabag/CoreBundle/fixtures/unnamed.png b/tests/Wallabag/CoreBundle/fixtures/unnamed.png new file mode 100644 index 00000000..e6dd9caa Binary files /dev/null and b/tests/Wallabag/CoreBundle/fixtures/unnamed.png differ diff --git a/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php index a3263771..856954a6 100644 --- a/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php +++ b/tests/Wallabag/ImportBundle/Consumer/AMQPEntryConsumerTest.php @@ -112,10 +112,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn($entry); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->once()) + ->method('dispatch'); + $consumer = new AMQPEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $message = new AMQPMessage($body); @@ -157,10 +166,19 @@ JSON; ->disableOriginalConstructor() ->getMock(); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new AMQPEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $message = new AMQPMessage($body); @@ -212,10 +230,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn(null); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new AMQPEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $message = new AMQPMessage($body); diff --git a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php index 01a92ad2..3b92f759 100644 --- a/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php +++ b/tests/Wallabag/ImportBundle/Consumer/RedisEntryConsumerTest.php @@ -111,10 +111,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn($entry); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->once()) + ->method('dispatch'); + $consumer = new RedisEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $res = $consumer->manage($body); @@ -156,10 +165,19 @@ JSON; ->disableOriginalConstructor() ->getMock(); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new RedisEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $res = $consumer->manage($body); @@ -211,10 +229,19 @@ JSON; ->with(json_decode($body, true)) ->willReturn(null); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->never()) + ->method('dispatch'); + $consumer = new RedisEntryConsumer( $em, $userRepository, - $import + $import, + $dispatcher ); $res = $consumer->manage($body); diff --git a/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php index 1e52615c..6b3adda4 100644 --- a/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php @@ -18,7 +18,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getChromeImport($unsetUser = false) + private function getChromeImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new ChromeImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new ChromeImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $chromeImport = $this->getChromeImport(); + $chromeImport = $this->getChromeImport(false, 1); $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class ChromeImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $chromeImport = $this->getChromeImport(); + $chromeImport = $this->getChromeImport(false, 1); $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php index 007dda6a..b516fbc5 100644 --- a/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php @@ -18,7 +18,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getFirefoxImport($unsetUser = false) + private function getFirefoxImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new FirefoxImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new FirefoxImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $firefoxImport = $this->getFirefoxImport(); + $firefoxImport = $this->getFirefoxImport(false, 2); $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class FirefoxImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $firefoxImport = $this->getFirefoxImport(); + $firefoxImport = $this->getFirefoxImport(false, 1); $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php b/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php index 75900bd7..e262a808 100644 --- a/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/InstapaperImportTest.php @@ -18,7 +18,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getInstapaperImport($unsetUser = false) + private function getInstapaperImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $import = new InstapaperImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $import = new InstapaperImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $instapaperImport = $this->getInstapaperImport(); + $instapaperImport = $this->getInstapaperImport(false, 3); $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class InstapaperImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $instapaperImport = $this->getInstapaperImport(); + $instapaperImport = $this->getInstapaperImport(false, 1); $instapaperImport->setFilepath(__DIR__.'/../fixtures/instapaper-export.csv'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php index 9ec7935c..141ece36 100644 --- a/tests/Wallabag/ImportBundle/Import/PocketImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/PocketImportTest.php @@ -24,7 +24,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase protected $contentProxy; protected $logHandler; - private function getPocketImport($consumerKey = 'ConsumerKey') + private function getPocketImport($consumerKey = 'ConsumerKey', $dispatched = 0) { $this->user = new User(); @@ -55,10 +55,15 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase ->method('getScheduledEntityInsertions') ->willReturn([]); - $pocket = new PocketImport( - $this->em, - $this->contentProxy - ); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $pocket = new PocketImport($this->em, $this->contentProxy, $dispatcher); $pocket->setUser($this->user); $this->logHandler = new TestHandler(); @@ -252,7 +257,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $client->getEmitter()->attach($mock); - $pocketImport = $this->getPocketImport(); + $pocketImport = $this->getPocketImport('ConsumerKey', 1); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') ->disableOriginalConstructor() @@ -339,7 +344,7 @@ class PocketImportTest extends \PHPUnit_Framework_TestCase $client->getEmitter()->attach($mock); - $pocketImport = $this->getPocketImport(); + $pocketImport = $this->getPocketImport('ConsumerKey', 2); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') ->disableOriginalConstructor() @@ -591,7 +596,7 @@ JSON; $client->getEmitter()->attach($mock); - $pocketImport = $this->getPocketImport(); + $pocketImport = $this->getPocketImport('ConsumerKey', 1); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') ->disableOriginalConstructor() diff --git a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php index d98cd486..d1bbe648 100644 --- a/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/ReadabilityImportTest.php @@ -18,7 +18,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getReadabilityImport($unsetUser = false) + private function getReadabilityImport($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -30,7 +30,15 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new ReadabilityImport($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new ReadabilityImport($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -54,7 +62,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $readabilityImport = $this->getReadabilityImport(); + $readabilityImport = $this->getReadabilityImport(false, 24); $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -87,7 +95,7 @@ class ReadabilityImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $readabilityImport = $this->getReadabilityImport(); + $readabilityImport = $this->getReadabilityImport(false, 1); $readabilityImport->setFilepath(__DIR__.'/../fixtures/readability-read.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php index 82dc4c7e..4dbced60 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV1ImportTest.php @@ -18,7 +18,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getWallabagV1Import($unsetUser = false) + private function getWallabagV1Import($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -44,7 +44,15 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new WallabagV1Import($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new WallabagV1Import($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -68,7 +76,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $wallabagV1Import = $this->getWallabagV1Import(); + $wallabagV1Import = $this->getWallabagV1Import(false, 3); $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -101,7 +109,7 @@ class WallabagV1ImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $wallabagV1Import = $this->getWallabagV1Import(); + $wallabagV1Import = $this->getWallabagV1Import(false, 3); $wallabagV1Import->setFilepath(__DIR__.'/../fixtures/wallabag-v1-read.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php index bea89efb..0e50b8b2 100644 --- a/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php +++ b/tests/Wallabag/ImportBundle/Import/WallabagV2ImportTest.php @@ -18,7 +18,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase protected $logHandler; protected $contentProxy; - private function getWallabagV2Import($unsetUser = false) + private function getWallabagV2Import($unsetUser = false, $dispatched = 0) { $this->user = new User(); @@ -44,7 +44,15 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase ->disableOriginalConstructor() ->getMock(); - $wallabag = new WallabagV2Import($this->em, $this->contentProxy); + $dispatcher = $this->getMockBuilder('Symfony\Component\EventDispatcher\EventDispatcher') + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $wallabag = new WallabagV2Import($this->em, $this->contentProxy, $dispatcher); $this->logHandler = new TestHandler(); $logger = new Logger('test', [$this->logHandler]); @@ -68,7 +76,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase public function testImport() { - $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import = $this->getWallabagV2Import(false, 2); $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -97,7 +105,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase public function testImportAndMarkAllAsRead() { - $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import = $this->getWallabagV2Import(false, 2); $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2-read.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') @@ -246,7 +254,7 @@ class WallabagV2ImportTest extends \PHPUnit_Framework_TestCase public function testImportWithExceptionFromGraby() { - $wallabagV2Import = $this->getWallabagV2Import(); + $wallabagV2Import = $this->getWallabagV2Import(false, 2); $wallabagV2Import->setFilepath(__DIR__.'/../fixtures/wallabag-v2.json'); $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') diff --git a/data/assets/.gitignore b/web/assets/images/.gitkeep similarity index 100% rename from data/assets/.gitignore rename to web/assets/images/.gitkeep