From 4dface66707688ea440a6a7569795a85631d1ff0 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Sun, 26 Jun 2016 10:27:31 +0200 Subject: first draft (from v1) --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 168 ++++++++++++++++++++++++ 1 file changed, 168 insertions(+) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 8019df42..ddeffa77 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -151,4 +151,172 @@ class ContentProxy { return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']); } + + /** + * Changing pictures URL in article content. + */ + public static function filterPicture($content, $url, $id) + { + $matches = array(); + $processing_pictures = array(); // list of processing image to avoid processing the same pictures twice + preg_match_all('#<\s*(img)[^>]+src="([^"]*)"[^>]*>#Si', $content, $matches, PREG_SET_ORDER); + foreach ($matches as $i => $link) { + $link[1] = trim($link[1]); + if (!preg_match('#^(([a-z]+://)|(\#))#', $link[1])) { + $absolute_path = self::_getAbsoluteLink($link[2], $url); + $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); + $directory = self::_createAssetsDirectory($id); + $fullpath = $directory.'/'.$filename; + + if (in_array($absolute_path, $processing_pictures) === true) { + // replace picture's URL only if processing is OK : already processing -> go to next picture + continue; + } + + if (self::_downloadPictures($absolute_path, $fullpath) === true) { + $content = str_replace($matches[$i][2], Tools::getPocheUrl().$fullpath, $content); + } + + $processing_pictures[] = $absolute_path; + } + } + + return $content; + } + + /** + * Get absolute URL. + */ + private static function _getAbsoluteLink($relativeLink, $url) + { + /* return if already absolute URL */ + if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { + return $relativeLink; + } + + /* queries and anchors */ + if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { + return $url.$relativeLink; + } + + /* parse base URL and convert to local variables: + $scheme, $host, $path */ + extract(parse_url($url)); + + /* remove non-directory element from path */ + $path = preg_replace('#/[^/]*$#', '', $path); + + /* destroy path if relative url points to root */ + if ($relativeLink[0] == '/') { + $path = ''; + } + + /* dirty absolute URL */ + $abs = $host.$path.'/'.$relativeLink; + + /* replace '//' or '/./' or '/foo/../' with '/' */ + $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); + for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { + } + + /* absolute URL is ready! */ + return $scheme.'://'.$abs; + } + + /** + * Downloading pictures. + * + * @return bool true if the download and processing is OK, false else + */ + private static function _downloadPictures($absolute_path, $fullpath) + { + $rawdata = Tools::getFile($absolute_path); + $fullpath = urldecode($fullpath); + + if (file_exists($fullpath)) { + unlink($fullpath); + } + + // check extension + $file_ext = strrchr($fullpath, '.'); + $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); + if (!(in_array($file_ext, $whitelist))) { + Tools::logm('processed image with not allowed extension. Skipping '.$fullpath); + + return false; + } + + // check headers + $imageinfo = getimagesize($absolute_path); + if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { + Tools::logm('processed image with bad header. Skipping '.$fullpath); + + return false; + } + + // regenerate image + $im = imagecreatefromstring($rawdata); + if ($im === false) { + Tools::logm('error while regenerating image '.$fullpath); + + return false; + } + + switch ($imageinfo['mime']) { + case 'image/gif': + $result = imagegif($im, $fullpath); + break; + case 'image/jpeg': + case 'image/jpg': + $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); + break; + case 'image/png': + $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); + break; + } + imagedestroy($im); + + return $result; + } + + /** + * Create a directory for an article. + * + * @param $id ID of the article + * + * @return string + */ + private static function _createAssetsDirectory($id) + { + $assets_path = ABS_PATH; + if (!is_dir($assets_path)) { + mkdir($assets_path, 0715); + } + + $article_directory = $assets_path.$id; + if (!is_dir($article_directory)) { + mkdir($article_directory, 0715); + } + + return $article_directory; + } + + /** + * Remove the directory. + * + * @param $directory + * + * @return bool + */ + public static function removeDirectory($directory) + { + if (is_dir($directory)) { + $files = array_diff(scandir($directory), array('.', '..')); + foreach ($files as $file) { + (is_dir("$directory/$file")) ? self::removeDirectory("$directory/$file") : unlink("$directory/$file"); + } + + return rmdir($directory); + } + } } -- cgit v1.2.3 From 419214d7221e0821ef2b73eb2b3db816ed0cf173 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 28 Jun 2016 19:07:55 +0200 Subject: Download pictures successfully Needs to rewrite them properly (get base url) --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 178 ++-------------------- src/Wallabag/CoreBundle/Helper/DownloadImages.php | 147 ++++++++++++++++++ 2 files changed, 156 insertions(+), 169 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Helper/DownloadImages.php (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index ddeffa77..bbad705f 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -65,7 +65,7 @@ class ContentProxy $entry->setUrl($content['url'] ?: $url); $entry->setTitle($title); - $entry->setContent($html); + $entry->setLanguage($content['language']); $entry->setMimetype($content['content_type']); $entry->setReadingTime(Utils::getReadingTime($html)); @@ -75,6 +75,14 @@ class ContentProxy $entry->setDomainName($domainName); } + if (true) { + $this->logger->log('debug','Starting to download images'); + $downloadImages = new DownloadImages($html, $url, $this->logger); + $html = $downloadImages->process(); + } + + $entry->setContent($html); + if (isset($content['open_graph']['og_image'])) { $entry->setPreviewPicture($content['open_graph']['og_image']); } @@ -151,172 +159,4 @@ class ContentProxy { return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']); } - - /** - * Changing pictures URL in article content. - */ - public static function filterPicture($content, $url, $id) - { - $matches = array(); - $processing_pictures = array(); // list of processing image to avoid processing the same pictures twice - preg_match_all('#<\s*(img)[^>]+src="([^"]*)"[^>]*>#Si', $content, $matches, PREG_SET_ORDER); - foreach ($matches as $i => $link) { - $link[1] = trim($link[1]); - if (!preg_match('#^(([a-z]+://)|(\#))#', $link[1])) { - $absolute_path = self::_getAbsoluteLink($link[2], $url); - $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); - $directory = self::_createAssetsDirectory($id); - $fullpath = $directory.'/'.$filename; - - if (in_array($absolute_path, $processing_pictures) === true) { - // replace picture's URL only if processing is OK : already processing -> go to next picture - continue; - } - - if (self::_downloadPictures($absolute_path, $fullpath) === true) { - $content = str_replace($matches[$i][2], Tools::getPocheUrl().$fullpath, $content); - } - - $processing_pictures[] = $absolute_path; - } - } - - return $content; - } - - /** - * Get absolute URL. - */ - private static function _getAbsoluteLink($relativeLink, $url) - { - /* return if already absolute URL */ - if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { - return $relativeLink; - } - - /* queries and anchors */ - if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { - return $url.$relativeLink; - } - - /* parse base URL and convert to local variables: - $scheme, $host, $path */ - extract(parse_url($url)); - - /* remove non-directory element from path */ - $path = preg_replace('#/[^/]*$#', '', $path); - - /* destroy path if relative url points to root */ - if ($relativeLink[0] == '/') { - $path = ''; - } - - /* dirty absolute URL */ - $abs = $host.$path.'/'.$relativeLink; - - /* replace '//' or '/./' or '/foo/../' with '/' */ - $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); - for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { - } - - /* absolute URL is ready! */ - return $scheme.'://'.$abs; - } - - /** - * Downloading pictures. - * - * @return bool true if the download and processing is OK, false else - */ - private static function _downloadPictures($absolute_path, $fullpath) - { - $rawdata = Tools::getFile($absolute_path); - $fullpath = urldecode($fullpath); - - if (file_exists($fullpath)) { - unlink($fullpath); - } - - // check extension - $file_ext = strrchr($fullpath, '.'); - $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); - if (!(in_array($file_ext, $whitelist))) { - Tools::logm('processed image with not allowed extension. Skipping '.$fullpath); - - return false; - } - - // check headers - $imageinfo = getimagesize($absolute_path); - if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { - Tools::logm('processed image with bad header. Skipping '.$fullpath); - - return false; - } - - // regenerate image - $im = imagecreatefromstring($rawdata); - if ($im === false) { - Tools::logm('error while regenerating image '.$fullpath); - - return false; - } - - switch ($imageinfo['mime']) { - case 'image/gif': - $result = imagegif($im, $fullpath); - break; - case 'image/jpeg': - case 'image/jpg': - $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); - break; - case 'image/png': - $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); - break; - } - imagedestroy($im); - - return $result; - } - - /** - * Create a directory for an article. - * - * @param $id ID of the article - * - * @return string - */ - private static function _createAssetsDirectory($id) - { - $assets_path = ABS_PATH; - if (!is_dir($assets_path)) { - mkdir($assets_path, 0715); - } - - $article_directory = $assets_path.$id; - if (!is_dir($article_directory)) { - mkdir($article_directory, 0715); - } - - return $article_directory; - } - - /** - * Remove the directory. - * - * @param $directory - * - * @return bool - */ - public static function removeDirectory($directory) - { - if (is_dir($directory)) { - $files = array_diff(scandir($directory), array('.', '..')); - foreach ($files as $file) { - (is_dir("$directory/$file")) ? self::removeDirectory("$directory/$file") : unlink("$directory/$file"); - } - - return rmdir($directory); - } - } } diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php new file mode 100644 index 00000000..32a9dbb2 --- /dev/null +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -0,0 +1,147 @@ +html = $html; + $this->url = $url; + $this->setFolder(); + $this->logger = $logger; + } + + public function setFolder($folder = "assets/images") { + // if folder doesn't exist, attempt to create one and store the folder name in property $folder + if(!file_exists($folder)) { + mkdir($folder); + } + $this->folder = $folder; + } + + public function process() { + //instantiate the symfony DomCrawler Component + $crawler = new Crawler($this->html); + // create an array of all scrapped image links + $this->logger->log('debug', 'Finding images inside document'); + $result = $crawler + ->filterXpath('//img') + ->extract(array('src')); + + // download and save the image to the folder + foreach ($result as $image) { + $file = file_get_contents($image); + + // Checks + $absolute_path = self::getAbsoluteLink($image, $this->url); + $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); + $fullpath = $this->folder."/".$filename; + self::checks($file, $fullpath, $absolute_path); + $this->html = str_replace($image, $fullpath, $this->html); + } + + return $this->html; + } + + private function checks($rawdata, $fullpath, $absolute_path) { + $fullpath = urldecode($fullpath); + + if (file_exists($fullpath)) { + unlink($fullpath); + } + + // check extension + $this->logger->log('debug','Checking extension'); + + $file_ext = strrchr($fullpath, '.'); + $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); + if (!(in_array($file_ext, $whitelist))) { + $this->logger->log('debug','processed image with not allowed extension. Skipping '.$fullpath); + + return false; + } + + // check headers + $this->logger->log('debug','Checking headers'); + $imageinfo = getimagesize($absolute_path); + if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { + $this->logger->log('debug','processed image with bad header. Skipping '.$fullpath); + + return false; + } + + // regenerate image + $this->logger->log('debug','regenerating image'); + $im = imagecreatefromstring($rawdata); + if ($im === false) { + $this->logger->log('error','error while regenerating image '.$fullpath); + + return false; + } + + switch ($imageinfo['mime']) { + case 'image/gif': + $result = imagegif($im, $fullpath); + $this->logger->log('debug','Re-creating gif'); + break; + case 'image/jpeg': + case 'image/jpg': + $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); + $this->logger->log('debug','Re-creating jpg'); + break; + case 'image/png': + $this->logger->log('debug','Re-creating png'); + $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); + break; + } + imagedestroy($im); + + return $result; + } + + private static function getAbsoluteLink($relativeLink, $url) + { + /* return if already absolute URL */ + if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { + return $relativeLink; + } + + /* queries and anchors */ + if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { + return $url.$relativeLink; + } + + /* parse base URL and convert to local variables: + $scheme, $host, $path */ + extract(parse_url($url)); + + /* remove non-directory element from path */ + $path = preg_replace('#/[^/]*$#', '', $path); + + /* destroy path if relative url points to root */ + if ($relativeLink[0] == '/') { + $path = ''; + } + + /* dirty absolute URL */ + $abs = $host.$path.'/'.$relativeLink; + + /* replace '//' or '/./' or '/foo/../' with '/' */ + $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); + for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { + } + + /* absolute URL is ready! */ + return $scheme.'://'.$abs; + } +} -- cgit v1.2.3 From 94654765cca6771c2f54eeaa056b7e65f3353105 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 28 Jun 2016 22:06:00 +0200 Subject: Working --- src/Wallabag/CoreBundle/Helper/DownloadImages.php | 44 ++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 32a9dbb2..14f0aa1b 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -6,6 +6,9 @@ use Psr\Log\LoggerInterface as Logger; use Symfony\Component\DomCrawler\Crawler; define('REGENERATE_PICTURES_QUALITY', 75); +define('HTTP_PORT', 80); +define('SSL_PORT', 443); +define('BASE_URL',''); class DownloadImages { private $folder; @@ -47,7 +50,7 @@ class DownloadImages { $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); $fullpath = $this->folder."/".$filename; self::checks($file, $fullpath, $absolute_path); - $this->html = str_replace($image, $fullpath, $this->html); + $this->html = str_replace($image, self::getPocheUrl() . '/' . $fullpath, $this->html); } return $this->html; @@ -144,4 +147,43 @@ class DownloadImages { /* absolute URL is ready! */ return $scheme.'://'.$abs; } + + public static function getPocheUrl() + { + $baseUrl = ""; + $https = (!empty($_SERVER['HTTPS']) + && (strtolower($_SERVER['HTTPS']) == 'on')) + || (isset($_SERVER["SERVER_PORT"]) + && $_SERVER["SERVER_PORT"] == '443') // HTTPS detection. + || (isset($_SERVER["SERVER_PORT"]) //Custom HTTPS port detection + && $_SERVER["SERVER_PORT"] == SSL_PORT) + || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) + && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https'); + $serverport = (!isset($_SERVER["SERVER_PORT"]) + || $_SERVER["SERVER_PORT"] == '80' + || $_SERVER["SERVER_PORT"] == HTTP_PORT + || ($https && $_SERVER["SERVER_PORT"] == '443') + || ($https && $_SERVER["SERVER_PORT"]==SSL_PORT) //Custom HTTPS port detection + ? '' : ':' . $_SERVER["SERVER_PORT"]); + + if (isset($_SERVER["HTTP_X_FORWARDED_PORT"])) { + $serverport = ':' . $_SERVER["HTTP_X_FORWARDED_PORT"]; + } + // $scriptname = str_replace('/index.php', '/', $_SERVER["SCRIPT_NAME"]); + // if (!isset($_SERVER["HTTP_HOST"])) { + // return $scriptname; + // } + $host = (isset($_SERVER['HTTP_X_FORWARDED_HOST']) ? $_SERVER['HTTP_X_FORWARDED_HOST'] : (isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : $_SERVER['SERVER_NAME'])); + if (strpos($host, ':') !== false) { + $serverport = ''; + } + // check if BASE_URL is configured + if(BASE_URL) { + $baseUrl = BASE_URL; + } else { + $baseUrl = 'http' . ($https ? 's' : '') . '://' . $host . $serverport; + } + return $baseUrl; + + } } -- cgit v1.2.3 From 156bf62758080153668a65db611c4241d0fc8a00 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sat, 22 Oct 2016 09:22:30 +0200 Subject: CS --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 2 +- src/Wallabag/CoreBundle/Helper/DownloadImages.php | 77 ++++++++++++----------- 2 files changed, 42 insertions(+), 37 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index bbad705f..8ed11205 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -76,7 +76,7 @@ class ContentProxy } if (true) { - $this->logger->log('debug','Starting to download images'); + $this->logger->log('debug', 'Starting to download images'); $downloadImages = new DownloadImages($html, $url, $this->logger); $html = $downloadImages->process(); } diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 14f0aa1b..e23e0c55 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -8,31 +8,35 @@ use Symfony\Component\DomCrawler\Crawler; define('REGENERATE_PICTURES_QUALITY', 75); define('HTTP_PORT', 80); define('SSL_PORT', 443); -define('BASE_URL',''); +define('BASE_URL', ''); -class DownloadImages { +class DownloadImages +{ private $folder; private $url; private $html; private $fileName; private $logger; - public function __construct($html, $url, Logger $logger) { + public function __construct($html, $url, Logger $logger) + { $this->html = $html; $this->url = $url; $this->setFolder(); $this->logger = $logger; } - public function setFolder($folder = "assets/images") { + public function setFolder($folder = 'assets/images') + { // if folder doesn't exist, attempt to create one and store the folder name in property $folder - if(!file_exists($folder)) { + if (!file_exists($folder)) { mkdir($folder); } $this->folder = $folder; } - public function process() { + public function process() + { //instantiate the symfony DomCrawler Component $crawler = new Crawler($this->html); // create an array of all scrapped image links @@ -48,15 +52,16 @@ class DownloadImages { // Checks $absolute_path = self::getAbsoluteLink($image, $this->url); $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); - $fullpath = $this->folder."/".$filename; + $fullpath = $this->folder.'/'.$filename; self::checks($file, $fullpath, $absolute_path); - $this->html = str_replace($image, self::getPocheUrl() . '/' . $fullpath, $this->html); + $this->html = str_replace($image, self::getPocheUrl().'/'.$fullpath, $this->html); } return $this->html; } - private function checks($rawdata, $fullpath, $absolute_path) { + private function checks($rawdata, $fullpath, $absolute_path) + { $fullpath = urldecode($fullpath); if (file_exists($fullpath)) { @@ -64,30 +69,30 @@ class DownloadImages { } // check extension - $this->logger->log('debug','Checking extension'); + $this->logger->log('debug', 'Checking extension'); $file_ext = strrchr($fullpath, '.'); $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); if (!(in_array($file_ext, $whitelist))) { - $this->logger->log('debug','processed image with not allowed extension. Skipping '.$fullpath); + $this->logger->log('debug', 'processed image with not allowed extension. Skipping '.$fullpath); return false; } // check headers - $this->logger->log('debug','Checking headers'); + $this->logger->log('debug', 'Checking headers'); $imageinfo = getimagesize($absolute_path); if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { - $this->logger->log('debug','processed image with bad header. Skipping '.$fullpath); + $this->logger->log('debug', 'processed image with bad header. Skipping '.$fullpath); return false; } // regenerate image - $this->logger->log('debug','regenerating image'); + $this->logger->log('debug', 'regenerating image'); $im = imagecreatefromstring($rawdata); if ($im === false) { - $this->logger->log('error','error while regenerating image '.$fullpath); + $this->logger->log('error', 'error while regenerating image '.$fullpath); return false; } @@ -95,15 +100,15 @@ class DownloadImages { switch ($imageinfo['mime']) { case 'image/gif': $result = imagegif($im, $fullpath); - $this->logger->log('debug','Re-creating gif'); + $this->logger->log('debug', 'Re-creating gif'); break; case 'image/jpeg': case 'image/jpg': $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); - $this->logger->log('debug','Re-creating jpg'); + $this->logger->log('debug', 'Re-creating jpg'); break; case 'image/png': - $this->logger->log('debug','Re-creating png'); + $this->logger->log('debug', 'Re-creating png'); $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); break; } @@ -150,24 +155,24 @@ class DownloadImages { public static function getPocheUrl() { - $baseUrl = ""; + $baseUrl = ''; $https = (!empty($_SERVER['HTTPS']) && (strtolower($_SERVER['HTTPS']) == 'on')) - || (isset($_SERVER["SERVER_PORT"]) - && $_SERVER["SERVER_PORT"] == '443') // HTTPS detection. - || (isset($_SERVER["SERVER_PORT"]) //Custom HTTPS port detection - && $_SERVER["SERVER_PORT"] == SSL_PORT) + || (isset($_SERVER['SERVER_PORT']) + && $_SERVER['SERVER_PORT'] == '443') // HTTPS detection. + || (isset($_SERVER['SERVER_PORT']) //Custom HTTPS port detection + && $_SERVER['SERVER_PORT'] == SSL_PORT) || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https'); - $serverport = (!isset($_SERVER["SERVER_PORT"]) - || $_SERVER["SERVER_PORT"] == '80' - || $_SERVER["SERVER_PORT"] == HTTP_PORT - || ($https && $_SERVER["SERVER_PORT"] == '443') - || ($https && $_SERVER["SERVER_PORT"]==SSL_PORT) //Custom HTTPS port detection - ? '' : ':' . $_SERVER["SERVER_PORT"]); - - if (isset($_SERVER["HTTP_X_FORWARDED_PORT"])) { - $serverport = ':' . $_SERVER["HTTP_X_FORWARDED_PORT"]; + $serverport = (!isset($_SERVER['SERVER_PORT']) + || $_SERVER['SERVER_PORT'] == '80' + || $_SERVER['SERVER_PORT'] == HTTP_PORT + || ($https && $_SERVER['SERVER_PORT'] == '443') + || ($https && $_SERVER['SERVER_PORT'] == SSL_PORT) //Custom HTTPS port detection + ? '' : ':'.$_SERVER['SERVER_PORT']); + + if (isset($_SERVER['HTTP_X_FORWARDED_PORT'])) { + $serverport = ':'.$_SERVER['HTTP_X_FORWARDED_PORT']; } // $scriptname = str_replace('/index.php', '/', $_SERVER["SCRIPT_NAME"]); // if (!isset($_SERVER["HTTP_HOST"])) { @@ -178,12 +183,12 @@ class DownloadImages { $serverport = ''; } // check if BASE_URL is configured - if(BASE_URL) { + if (BASE_URL) { $baseUrl = BASE_URL; } else { - $baseUrl = 'http' . ($https ? 's' : '') . '://' . $host . $serverport; + $baseUrl = 'http'.($https ? 's' : '').'://'.$host.$serverport; } - return $baseUrl; - + + return $baseUrl; } } -- cgit v1.2.3 From 535bfcbe80de5d697b768c3a657214fdeff0eac3 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 09:58:39 +0100 Subject: Move related event things in Event folder --- .../CoreBundle/Event/Listener/LocaleListener.php | 44 ++++++++++++++ .../Event/Listener/UserLocaleListener.php | 37 ++++++++++++ .../Subscriber/SQLiteCascadeDeleteSubscriber.php | 70 ++++++++++++++++++++++ .../Event/Subscriber/TablePrefixSubscriber.php | 51 ++++++++++++++++ .../CoreBundle/EventListener/LocaleListener.php | 44 -------------- .../EventListener/UserLocaleListener.php | 37 ------------ .../CoreBundle/Resources/config/services.yml | 4 +- .../Subscriber/SQLiteCascadeDeleteSubscriber.php | 70 ---------------------- .../Subscriber/TablePrefixSubscriber.php | 51 ---------------- 9 files changed, 204 insertions(+), 204 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php create mode 100644 src/Wallabag/CoreBundle/Event/Listener/UserLocaleListener.php create mode 100644 src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php create mode 100644 src/Wallabag/CoreBundle/Event/Subscriber/TablePrefixSubscriber.php delete mode 100644 src/Wallabag/CoreBundle/EventListener/LocaleListener.php delete mode 100644 src/Wallabag/CoreBundle/EventListener/UserLocaleListener.php delete mode 100644 src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php delete mode 100644 src/Wallabag/CoreBundle/Subscriber/TablePrefixSubscriber.php (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php b/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php new file mode 100644 index 00000000..b435d99e --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/Listener/LocaleListener.php @@ -0,0 +1,44 @@ +defaultLocale = $defaultLocale; + } + + public function onKernelRequest(GetResponseEvent $event) + { + $request = $event->getRequest(); + if (!$request->hasPreviousSession()) { + return; + } + + // try to see if the locale has been set as a _locale routing parameter + if ($locale = $request->attributes->get('_locale')) { + $request->getSession()->set('_locale', $locale); + } else { + // if no explicit locale has been set on this request, use one from the session + $request->setLocale($request->getSession()->get('_locale', $this->defaultLocale)); + } + } + + public static function getSubscribedEvents() + { + return [ + // must be registered before the default Locale listener + KernelEvents::REQUEST => [['onKernelRequest', 17]], + ]; + } +} diff --git a/src/Wallabag/CoreBundle/Event/Listener/UserLocaleListener.php b/src/Wallabag/CoreBundle/Event/Listener/UserLocaleListener.php new file mode 100644 index 00000000..367cdfb0 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/Listener/UserLocaleListener.php @@ -0,0 +1,37 @@ +session = $session; + } + + /** + * @param InteractiveLoginEvent $event + */ + public function onInteractiveLogin(InteractiveLoginEvent $event) + { + $user = $event->getAuthenticationToken()->getUser(); + + if (null !== $user->getConfig()->getLanguage()) { + $this->session->set('_locale', $user->getConfig()->getLanguage()); + } + } +} diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php new file mode 100644 index 00000000..3b4c4cf9 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/Subscriber/SQLiteCascadeDeleteSubscriber.php @@ -0,0 +1,70 @@ +doctrine = $doctrine; + } + + /** + * @return array + */ + public function getSubscribedEvents() + { + return [ + 'preRemove', + ]; + } + + /** + * We removed everything related to the upcoming removed entry because SQLite can't handle it on it own. + * We do it in the preRemove, because we can't retrieve tags in the postRemove (because the entry id is gone). + * + * @param LifecycleEventArgs $args + */ + public function preRemove(LifecycleEventArgs $args) + { + $entity = $args->getEntity(); + + if (!$this->doctrine->getConnection()->getDriver() instanceof \Doctrine\DBAL\Driver\PDOSqlite\Driver || + !$entity instanceof Entry) { + return; + } + + $em = $this->doctrine->getManager(); + + if (null !== $entity->getTags()) { + foreach ($entity->getTags() as $tag) { + $entity->removeTag($tag); + } + } + + if (null !== $entity->getAnnotations()) { + foreach ($entity->getAnnotations() as $annotation) { + $em->remove($annotation); + } + } + + $em->flush(); + } +} diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/TablePrefixSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/TablePrefixSubscriber.php new file mode 100644 index 00000000..9013328f --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/Subscriber/TablePrefixSubscriber.php @@ -0,0 +1,51 @@ +prefix = (string) $prefix; + } + + public function getSubscribedEvents() + { + return ['loadClassMetadata']; + } + + public function loadClassMetadata(LoadClassMetadataEventArgs $args) + { + $classMetadata = $args->getClassMetadata(); + + // if we are in an inheritance hierarchy, only apply this once + if ($classMetadata->isInheritanceTypeSingleTable() && !$classMetadata->isRootEntity()) { + return; + } + + $classMetadata->setTableName($this->prefix.$classMetadata->getTableName()); + + foreach ($classMetadata->getAssociationMappings() as $fieldName => $mapping) { + if ($mapping['type'] === ClassMetadataInfo::MANY_TO_MANY && isset($classMetadata->associationMappings[$fieldName]['joinTable']['name'])) { + $mappedTableName = $classMetadata->associationMappings[$fieldName]['joinTable']['name']; + $classMetadata->associationMappings[$fieldName]['joinTable']['name'] = $this->prefix.$mappedTableName; + } + } + } +} diff --git a/src/Wallabag/CoreBundle/EventListener/LocaleListener.php b/src/Wallabag/CoreBundle/EventListener/LocaleListener.php deleted file mode 100644 index a1c7e5ab..00000000 --- a/src/Wallabag/CoreBundle/EventListener/LocaleListener.php +++ /dev/null @@ -1,44 +0,0 @@ -defaultLocale = $defaultLocale; - } - - public function onKernelRequest(GetResponseEvent $event) - { - $request = $event->getRequest(); - if (!$request->hasPreviousSession()) { - return; - } - - // try to see if the locale has been set as a _locale routing parameter - if ($locale = $request->attributes->get('_locale')) { - $request->getSession()->set('_locale', $locale); - } else { - // if no explicit locale has been set on this request, use one from the session - $request->setLocale($request->getSession()->get('_locale', $this->defaultLocale)); - } - } - - public static function getSubscribedEvents() - { - return [ - // must be registered before the default Locale listener - KernelEvents::REQUEST => [['onKernelRequest', 17]], - ]; - } -} diff --git a/src/Wallabag/CoreBundle/EventListener/UserLocaleListener.php b/src/Wallabag/CoreBundle/EventListener/UserLocaleListener.php deleted file mode 100644 index 82d1a63a..00000000 --- a/src/Wallabag/CoreBundle/EventListener/UserLocaleListener.php +++ /dev/null @@ -1,37 +0,0 @@ -session = $session; - } - - /** - * @param InteractiveLoginEvent $event - */ - public function onInteractiveLogin(InteractiveLoginEvent $event) - { - $user = $event->getAuthenticationToken()->getUser(); - - if (null !== $user->getConfig()->getLanguage()) { - $this->session->set('_locale', $user->getConfig()->getLanguage()); - } - } -} diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index cc5f9e9a..4b7751fe 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -30,7 +30,7 @@ services: - "@doctrine" wallabag_core.subscriber.table_prefix: - class: Wallabag\CoreBundle\Subscriber\TablePrefixSubscriber + class: Wallabag\CoreBundle\Event\Subscriber\TablePrefixSubscriber arguments: - "%database_table_prefix%" tags: @@ -131,7 +131,7 @@ services: - '%kernel.debug%' wallabag_core.subscriber.sqlite_cascade_delete: - class: Wallabag\CoreBundle\Subscriber\SQLiteCascadeDeleteSubscriber + class: Wallabag\CoreBundle\Event\Subscriber\SQLiteCascadeDeleteSubscriber arguments: - "@doctrine" tags: diff --git a/src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php b/src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php deleted file mode 100644 index f7210bd3..00000000 --- a/src/Wallabag/CoreBundle/Subscriber/SQLiteCascadeDeleteSubscriber.php +++ /dev/null @@ -1,70 +0,0 @@ -doctrine = $doctrine; - } - - /** - * @return array - */ - public function getSubscribedEvents() - { - return [ - 'preRemove', - ]; - } - - /** - * We removed everything related to the upcoming removed entry because SQLite can't handle it on it own. - * We do it in the preRemove, because we can't retrieve tags in the postRemove (because the entry id is gone). - * - * @param LifecycleEventArgs $args - */ - public function preRemove(LifecycleEventArgs $args) - { - $entity = $args->getEntity(); - - if (!$this->doctrine->getConnection()->getDriver() instanceof \Doctrine\DBAL\Driver\PDOSqlite\Driver || - !$entity instanceof Entry) { - return; - } - - $em = $this->doctrine->getManager(); - - if (null !== $entity->getTags()) { - foreach ($entity->getTags() as $tag) { - $entity->removeTag($tag); - } - } - - if (null !== $entity->getAnnotations()) { - foreach ($entity->getAnnotations() as $annotation) { - $em->remove($annotation); - } - } - - $em->flush(); - } -} diff --git a/src/Wallabag/CoreBundle/Subscriber/TablePrefixSubscriber.php b/src/Wallabag/CoreBundle/Subscriber/TablePrefixSubscriber.php deleted file mode 100644 index 0379ad6a..00000000 --- a/src/Wallabag/CoreBundle/Subscriber/TablePrefixSubscriber.php +++ /dev/null @@ -1,51 +0,0 @@ -prefix = (string) $prefix; - } - - public function getSubscribedEvents() - { - return ['loadClassMetadata']; - } - - public function loadClassMetadata(LoadClassMetadataEventArgs $args) - { - $classMetadata = $args->getClassMetadata(); - - // if we are in an inheritance hierarchy, only apply this once - if ($classMetadata->isInheritanceTypeSingleTable() && !$classMetadata->isRootEntity()) { - return; - } - - $classMetadata->setTableName($this->prefix.$classMetadata->getTableName()); - - foreach ($classMetadata->getAssociationMappings() as $fieldName => $mapping) { - if ($mapping['type'] === ClassMetadataInfo::MANY_TO_MANY && isset($classMetadata->associationMappings[$fieldName]['joinTable']['name'])) { - $mappedTableName = $classMetadata->associationMappings[$fieldName]['joinTable']['name']; - $classMetadata->associationMappings[$fieldName]['joinTable']['name'] = $this->prefix.$mappedTableName; - } - } - } -} -- cgit v1.2.3 From 45fd7e09d75995bd0b9a731ffd70054b7ae6ee1f Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 09:58:53 +0100 Subject: Cleanup --- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 4 ++-- src/Wallabag/ImportBundle/Resources/config/services.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 8ed11205..219b90d3 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -3,7 +3,7 @@ namespace Wallabag\CoreBundle\Helper; use Graby\Graby; -use Psr\Log\LoggerInterface as Logger; +use Psr\Log\LoggerInterface; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\CoreBundle\Tools\Utils; @@ -20,7 +20,7 @@ class ContentProxy protected $logger; protected $tagRepository; - public function __construct(Graby $graby, RuleBasedTagger $tagger, TagRepository $tagRepository, Logger $logger) + public function __construct(Graby $graby, RuleBasedTagger $tagger, TagRepository $tagRepository, LoggerInterface $logger) { $this->graby = $graby; $this->tagger = $tagger; diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 89adc71b..d600be0f 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -20,7 +20,6 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" - - "@craue_config" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] -- cgit v1.2.3 From 7f55941856549a3f5f45c42fdc171d66ff7ee297 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 10:48:29 +0100 Subject: Use doctrine event to download images --- .../Event/Subscriber/DownloadImagesSubscriber.php | 129 +++++++++++ src/Wallabag/CoreBundle/Helper/ContentProxy.php | 6 - src/Wallabag/CoreBundle/Helper/DownloadImages.php | 248 +++++++++++---------- .../CoreBundle/Resources/config/services.yml | 19 ++ 4 files changed, 274 insertions(+), 128 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php new file mode 100644 index 00000000..654edf31 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -0,0 +1,129 @@ +downloadImages = $downloadImages; + $this->configClass = $configClass; + $this->logger = $logger; + } + + public function getSubscribedEvents() + { + return array( + 'prePersist', + 'preUpdate', + ); + } + + /** + * In case of an entry has been updated. + * We won't update the content field if it wasn't updated. + * + * @param LifecycleEventArgs $args + */ + public function preUpdate(LifecycleEventArgs $args) + { + $entity = $args->getEntity(); + + if (!$entity instanceof Entry) { + return; + } + + $em = $args->getEntityManager(); + + // field content has been updated + if ($args->hasChangedField('content')) { + $html = $this->downloadImages($em, $entity); + + if (null !== $html) { + $args->setNewValue('content', $html); + } + } + + // field preview picture has been updated + if ($args->hasChangedField('previewPicture')) { + $previewPicture = $this->downloadPreviewImage($em, $entity); + + if (null !== $previewPicture) { + $entity->setPreviewPicture($previewPicture); + } + } + } + + /** + * When a new entry is saved. + * + * @param LifecycleEventArgs $args + */ + public function prePersist(LifecycleEventArgs $args) + { + $entity = $args->getEntity(); + + if (!$entity instanceof Entry) { + return; + } + + $config = new $this->configClass(); + $config->setEntityManager($args->getEntityManager()); + + // update all images inside the html + $html = $this->downloadImages($config, $entity); + if (null !== $html) { + $entity->setContent($html); + } + + // update preview picture + $previewPicture = $this->downloadPreviewImage($config, $entity); + if (null !== $previewPicture) { + $entity->setPreviewPicture($previewPicture); + } + } + + public function downloadImages(Config $config, Entry $entry) + { + // if ($config->get('download_images_with_rabbitmq')) { + + // } else if ($config->get('download_images_with_redis')) { + + // } + + return $this->downloadImages->processHtml( + $entry->getContent(), + $entry->getUrl() + ); + } + + public function downloadPreviewImage(Config $config, Entry $entry) + { + // if ($config->get('download_images_with_rabbitmq')) { + + // } else if ($config->get('download_images_with_redis')) { + + // } + + return $this->downloadImages->processSingleImage( + $entry->getPreviewPicture(), + $entry->getUrl() + ); + } +} diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index 219b90d3..d90d3dc8 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -75,12 +75,6 @@ class ContentProxy $entry->setDomainName($domainName); } - if (true) { - $this->logger->log('debug', 'Starting to download images'); - $downloadImages = new DownloadImages($html, $url, $this->logger); - $html = $downloadImages->process(); - } - $entry->setContent($html); if (isset($content['open_graph']['og_image'])) { diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index e23e0c55..426cbe48 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -2,193 +2,197 @@ namespace Wallabag\CoreBundle\Helper; -use Psr\Log\LoggerInterface as Logger; +use Psr\Log\LoggerInterface; use Symfony\Component\DomCrawler\Crawler; - -define('REGENERATE_PICTURES_QUALITY', 75); -define('HTTP_PORT', 80); -define('SSL_PORT', 443); -define('BASE_URL', ''); +use GuzzleHttp\Client; +use Symfony\Component\HttpFoundation\File\MimeType\MimeTypeExtensionGuesser; class DownloadImages { - private $folder; - private $url; - private $html; - private $fileName; + const REGENERATE_PICTURES_QUALITY = 80; + + private $client; + private $baseFolder; private $logger; + private $mimeGuesser; - public function __construct($html, $url, Logger $logger) + public function __construct(Client $client, $baseFolder, LoggerInterface $logger) { - $this->html = $html; - $this->url = $url; - $this->setFolder(); + $this->client = $client; + $this->baseFolder = $baseFolder; $this->logger = $logger; + $this->mimeGuesser = new MimeTypeExtensionGuesser(); + + $this->setFolder(); } - public function setFolder($folder = 'assets/images') + /** + * Setup base folder where all images are going to be saved. + */ + private function setFolder() { // if folder doesn't exist, attempt to create one and store the folder name in property $folder - if (!file_exists($folder)) { - mkdir($folder); + if (!file_exists($this->baseFolder)) { + mkdir($this->baseFolder, 0777, true); } - $this->folder = $folder; } - public function process() + /** + * Process the html and extract image from it, save them to local and return the updated html. + * + * @param string $html + * @param string $url Used as a base path for relative image and folder + * + * @return string + */ + public function processHtml($html, $url) { - //instantiate the symfony DomCrawler Component - $crawler = new Crawler($this->html); - // create an array of all scrapped image links - $this->logger->log('debug', 'Finding images inside document'); + $crawler = new Crawler($html); $result = $crawler ->filterXpath('//img') ->extract(array('src')); + $relativePath = $this->getRelativePath($url); + // download and save the image to the folder foreach ($result as $image) { - $file = file_get_contents($image); - - // Checks - $absolute_path = self::getAbsoluteLink($image, $this->url); - $filename = basename(parse_url($absolute_path, PHP_URL_PATH)); - $fullpath = $this->folder.'/'.$filename; - self::checks($file, $fullpath, $absolute_path); - $this->html = str_replace($image, self::getPocheUrl().'/'.$fullpath, $this->html); + $imagePath = $this->processSingleImage($image, $url, $relativePath); + + if (false === $imagePath) { + continue; + } + + $html = str_replace($image, $imagePath, $html); } - return $this->html; + return $html; } - private function checks($rawdata, $fullpath, $absolute_path) + /** + * Process a single image: + * - retrieve it + * - re-saved it (for security reason) + * - return the new local path. + * + * @param string $imagePath Path to the image to retrieve + * @param string $url Url from where the image were found + * @param string $relativePath Relative local path to saved the image + * + * @return string Relative url to access the image from the web + */ + public function processSingleImage($imagePath, $url, $relativePath = null) { - $fullpath = urldecode($fullpath); - - if (file_exists($fullpath)) { - unlink($fullpath); + if (null == $relativePath) { + $relativePath = $this->getRelativePath($url); } - // check extension - $this->logger->log('debug', 'Checking extension'); + $folderPath = $this->baseFolder.'/'.$relativePath; - $file_ext = strrchr($fullpath, '.'); - $whitelist = array('.jpg', '.jpeg', '.gif', '.png'); - if (!(in_array($file_ext, $whitelist))) { - $this->logger->log('debug', 'processed image with not allowed extension. Skipping '.$fullpath); + // build image path + $absolutePath = $this->getAbsoluteLink($url, $imagePath); + if (false === $absolutePath) { + $this->logger->log('debug', 'Can not determine the absolute path for that image, skipping.'); return false; } - // check headers - $this->logger->log('debug', 'Checking headers'); - $imageinfo = getimagesize($absolute_path); - if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') { - $this->logger->log('debug', 'processed image with bad header. Skipping '.$fullpath); + $res = $this->client->get( + $absolutePath, + ['exceptions' => false] + ); + + $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); + $this->logger->log('debug', 'Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); + if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'])) { + $this->logger->log('debug', 'Processed image with not allowed extension. Skipping '.$imagePath); return false; } + $hashImage = hash('crc32', $absolutePath); + $localPath = $folderPath.'/'.$hashImage.'.'.$ext; + + try { + $im = imagecreatefromstring($res->getBody()); + } catch (\Exception $e) { + $im = false; + } - // regenerate image - $this->logger->log('debug', 'regenerating image'); - $im = imagecreatefromstring($rawdata); if ($im === false) { - $this->logger->log('error', 'error while regenerating image '.$fullpath); + $this->logger->log('error', 'Error while regenerating image', ['path' => $localPath]); return false; } - switch ($imageinfo['mime']) { - case 'image/gif': - $result = imagegif($im, $fullpath); + switch ($ext) { + case 'gif': + $result = imagegif($im, $localPath); $this->logger->log('debug', 'Re-creating gif'); break; - case 'image/jpeg': - case 'image/jpg': - $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY); + case 'jpeg': + case 'jpg': + $result = imagejpeg($im, $localPath, self::REGENERATE_PICTURES_QUALITY); $this->logger->log('debug', 'Re-creating jpg'); break; - case 'image/png': + case 'png': + $result = imagepng($im, $localPath, ceil(self::REGENERATE_PICTURES_QUALITY / 100 * 9)); $this->logger->log('debug', 'Re-creating png'); - $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9)); - break; } + imagedestroy($im); - return $result; + return '/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; } - private static function getAbsoluteLink($relativeLink, $url) + /** + * Generate the folder where we are going to save images based on the entry url. + * + * @param string $url + * + * @return string + */ + private function getRelativePath($url) { - /* return if already absolute URL */ - if (parse_url($relativeLink, PHP_URL_SCHEME) != '') { - return $relativeLink; - } + $hashUrl = hash('crc32', $url); + $relativePath = $hashUrl[0].'/'.$hashUrl[1].'/'.$hashUrl; + $folderPath = $this->baseFolder.'/'.$relativePath; - /* queries and anchors */ - if ($relativeLink[0] == '#' || $relativeLink[0] == '?') { - return $url.$relativeLink; + if (!file_exists($folderPath)) { + mkdir($folderPath, 0777, true); } - /* parse base URL and convert to local variables: - $scheme, $host, $path */ - extract(parse_url($url)); + $this->logger->log('debug', 'Folder used for that url', ['folder' => $folderPath, 'url' => $url]); - /* remove non-directory element from path */ - $path = preg_replace('#/[^/]*$#', '', $path); + return $relativePath; + } - /* destroy path if relative url points to root */ - if ($relativeLink[0] == '/') { - $path = ''; + /** + * Make an $url absolute based on the $base. + * + * @see Graby->makeAbsoluteStr + * + * @param string $base Base url + * @param string $url Url to make it absolute + * + * @return false|string + */ + private function getAbsoluteLink($base, $url) + { + if (preg_match('!^https?://!i', $url)) { + // already absolute + return $url; } - /* dirty absolute URL */ - $abs = $host.$path.'/'.$relativeLink; + $base = new \SimplePie_IRI($base); - /* replace '//' or '/./' or '/foo/../' with '/' */ - $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#'); - for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) { + // remove '//' in URL path (causes URLs not to resolve properly) + if (isset($base->ipath)) { + $base->ipath = preg_replace('!//+!', '/', $base->ipath); } - /* absolute URL is ready! */ - return $scheme.'://'.$abs; - } - - public static function getPocheUrl() - { - $baseUrl = ''; - $https = (!empty($_SERVER['HTTPS']) - && (strtolower($_SERVER['HTTPS']) == 'on')) - || (isset($_SERVER['SERVER_PORT']) - && $_SERVER['SERVER_PORT'] == '443') // HTTPS detection. - || (isset($_SERVER['SERVER_PORT']) //Custom HTTPS port detection - && $_SERVER['SERVER_PORT'] == SSL_PORT) - || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) - && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https'); - $serverport = (!isset($_SERVER['SERVER_PORT']) - || $_SERVER['SERVER_PORT'] == '80' - || $_SERVER['SERVER_PORT'] == HTTP_PORT - || ($https && $_SERVER['SERVER_PORT'] == '443') - || ($https && $_SERVER['SERVER_PORT'] == SSL_PORT) //Custom HTTPS port detection - ? '' : ':'.$_SERVER['SERVER_PORT']); - - if (isset($_SERVER['HTTP_X_FORWARDED_PORT'])) { - $serverport = ':'.$_SERVER['HTTP_X_FORWARDED_PORT']; - } - // $scriptname = str_replace('/index.php', '/', $_SERVER["SCRIPT_NAME"]); - // if (!isset($_SERVER["HTTP_HOST"])) { - // return $scriptname; - // } - $host = (isset($_SERVER['HTTP_X_FORWARDED_HOST']) ? $_SERVER['HTTP_X_FORWARDED_HOST'] : (isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : $_SERVER['SERVER_NAME'])); - if (strpos($host, ':') !== false) { - $serverport = ''; - } - // check if BASE_URL is configured - if (BASE_URL) { - $baseUrl = BASE_URL; - } else { - $baseUrl = 'http'.($https ? 's' : '').'://'.$host.$serverport; + if ($absolute = \SimplePie_IRI::absolutize($base, $url)) { + return $absolute->get_uri(); } - return $baseUrl; + return false; } } diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index 4b7751fe..1fb81a46 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -136,3 +136,22 @@ services: - "@doctrine" tags: - { name: doctrine.event_subscriber } + + wallabag_core.subscriber.download_images: + class: Wallabag\CoreBundle\Event\Subscriber\DownloadImagesSubscriber + arguments: + - "@wallabag_core.entry.download_images" + - "%craue_config.config.class%" + - "@logger" + tags: + - { name: doctrine.event_subscriber } + + wallabag_core.entry.download_images: + class: Wallabag\CoreBundle\Helper\DownloadImages + arguments: + - "@wallabag_core.entry.download_images.client" + - "%kernel.root_dir%/../web/assets/images" + - "@logger" + + wallabag_core.entry.download_images.client: + class: GuzzleHttp\Client -- cgit v1.2.3 From 48656e0eaac006a80f21e9aec8900747fe76283a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 11:27:09 +0100 Subject: Fixing tests --- .../Event/Subscriber/DownloadImagesSubscriber.php | 31 +++++++++++++++++----- src/Wallabag/CoreBundle/Helper/ContentProxy.php | 3 +-- src/Wallabag/CoreBundle/Helper/DownloadImages.php | 21 +++++++++------ 3 files changed, 38 insertions(+), 17 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 654edf31..09f8e911 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -49,22 +49,23 @@ class DownloadImagesSubscriber implements EventSubscriber return; } - $em = $args->getEntityManager(); + $config = new $this->configClass(); + $config->setEntityManager($args->getEntityManager()); // field content has been updated if ($args->hasChangedField('content')) { - $html = $this->downloadImages($em, $entity); + $html = $this->downloadImages($config, $entity); - if (null !== $html) { + if (false !== $html) { $args->setNewValue('content', $html); } } // field preview picture has been updated if ($args->hasChangedField('previewPicture')) { - $previewPicture = $this->downloadPreviewImage($em, $entity); + $previewPicture = $this->downloadPreviewImage($config, $entity); - if (null !== $previewPicture) { + if (false !== $previewPicture) { $entity->setPreviewPicture($previewPicture); } } @@ -88,17 +89,25 @@ class DownloadImagesSubscriber implements EventSubscriber // update all images inside the html $html = $this->downloadImages($config, $entity); - if (null !== $html) { + if (false !== $html) { $entity->setContent($html); } // update preview picture $previewPicture = $this->downloadPreviewImage($config, $entity); - if (null !== $previewPicture) { + if (false !== $previewPicture) { $entity->setPreviewPicture($previewPicture); } } + /** + * Download all images from the html. + * + * @param Config $config + * @param Entry $entry + * + * @return string|false False in case of async + */ public function downloadImages(Config $config, Entry $entry) { // if ($config->get('download_images_with_rabbitmq')) { @@ -113,6 +122,14 @@ class DownloadImagesSubscriber implements EventSubscriber ); } + /** + * Download the preview picture. + * + * @param Config $config + * @param Entry $entry + * + * @return string|false False in case of async + */ public function downloadPreviewImage(Config $config, Entry $entry) { // if ($config->get('download_images_with_rabbitmq')) { diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php index d90d3dc8..1986ab33 100644 --- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php +++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php @@ -65,6 +65,7 @@ class ContentProxy $entry->setUrl($content['url'] ?: $url); $entry->setTitle($title); + $entry->setContent($html); $entry->setLanguage($content['language']); $entry->setMimetype($content['content_type']); @@ -75,8 +76,6 @@ class ContentProxy $entry->setDomainName($domainName); } - $entry->setContent($html); - if (isset($content['open_graph']['og_image'])) { $entry->setPreviewPicture($content['open_graph']['og_image']); } diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 426cbe48..004bb277 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -91,20 +91,23 @@ class DownloadImages // build image path $absolutePath = $this->getAbsoluteLink($url, $imagePath); if (false === $absolutePath) { - $this->logger->log('debug', 'Can not determine the absolute path for that image, skipping.'); + $this->logger->log('error', 'Can not determine the absolute path for that image, skipping.'); return false; } - $res = $this->client->get( - $absolutePath, - ['exceptions' => false] - ); + try { + $res = $this->client->get($absolutePath); + } catch (\Exception $e) { + $this->logger->log('error', 'Can not retrieve image, skipping.', ['exception' => $e]); + + return false; + } $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); $this->logger->log('debug', 'Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); - if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'])) { - $this->logger->log('debug', 'Processed image with not allowed extension. Skipping '.$imagePath); + if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { + $this->logger->log('error', 'Processed image with not allowed extension. Skipping '.$imagePath); return false; } @@ -117,7 +120,7 @@ class DownloadImages $im = false; } - if ($im === false) { + if (false === $im) { $this->logger->log('error', 'Error while regenerating image', ['path' => $localPath]); return false; @@ -193,6 +196,8 @@ class DownloadImages return $absolute->get_uri(); } + $this->logger->log('error', 'Can not make an absolute link', ['base' => $base, 'url' => $url]); + return false; } } -- cgit v1.2.3 From 41ada277f066ea57947bce05bcda63962b7fea55 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 19:50:00 +0100 Subject: Add instance url to the downloaded images --- .../Event/Subscriber/DownloadImagesSubscriber.php | 4 ++++ src/Wallabag/CoreBundle/Helper/DownloadImages.php | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 09f8e911..0792653e 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -110,6 +110,8 @@ class DownloadImagesSubscriber implements EventSubscriber */ public function downloadImages(Config $config, Entry $entry) { + $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); + // if ($config->get('download_images_with_rabbitmq')) { // } else if ($config->get('download_images_with_redis')) { @@ -132,6 +134,8 @@ class DownloadImagesSubscriber implements EventSubscriber */ public function downloadPreviewImage(Config $config, Entry $entry) { + $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); + // if ($config->get('download_images_with_rabbitmq')) { // } else if ($config->get('download_images_with_redis')) { diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index 004bb277..e7982c56 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -15,6 +15,7 @@ class DownloadImages private $baseFolder; private $logger; private $mimeGuesser; + private $wallabagUrl; public function __construct(Client $client, $baseFolder, LoggerInterface $logger) { @@ -26,6 +27,17 @@ class DownloadImages $this->setFolder(); } + /** + * Since we can't inject CraueConfig service because it'll generate a circular reference when injected in the subscriber + * we use a different way to inject the current wallabag url. + * + * @param string $url Usually from `$config->get('wallabag_url')` + */ + public function setWallabagUrl($url) + { + $this->wallabagUrl = rtrim($url, '/'); + } + /** * Setup base folder where all images are going to be saved. */ @@ -143,7 +155,7 @@ class DownloadImages imagedestroy($im); - return '/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; + return $this->wallabagUrl.'/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; } /** -- cgit v1.2.3 From 309e13c11b54277626f18616c41f68ae9656a403 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 20:12:34 +0100 Subject: Move settings before Entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because we need wallabag_url to be defined when we’ll insert entries --- src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index a5e1be65..12f66c19 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -158,6 +158,6 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface */ public function getOrder() { - return 50; + return 29; } } -- cgit v1.2.3 From d1495dd0a456f0e35a09fb68679ee51f8d17bfe4 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Sun, 30 Oct 2016 21:30:45 +0100 Subject: Ability to enable/disable downloading images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will speed up the test suite because it won’t download everything when we add new entry… Add a custom test with downloading image enabled --- src/Wallabag/CoreBundle/Command/InstallCommand.php | 15 +++++++++++++++ .../CoreBundle/DataFixtures/ORM/LoadSettingData.php | 15 +++++++++++++++ .../Event/Subscriber/DownloadImagesSubscriber.php | 8 ++++++++ 3 files changed, 38 insertions(+) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index 277f8524..aedccfe4 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -398,6 +398,21 @@ class InstallCommand extends ContainerAwareCommand 'value' => 'wallabag', 'section' => 'misc', ], + [ + 'name' => 'download_images_enabled', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_rabbitmq', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_redis', + 'value' => '0', + 'section' => 'image', + ], ]; foreach ($settings as $setting) { diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index 12f66c19..70a7a4ac 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -140,6 +140,21 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface 'value' => 'wallabag', 'section' => 'misc', ], + [ + 'name' => 'download_images_enabled', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_rabbitmq', + 'value' => '0', + 'section' => 'image', + ], + [ + 'name' => 'download_images_with_redis', + 'value' => '0', + 'section' => 'image', + ], ]; foreach ($settings as $setting) { diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 0792653e..3f2d460c 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -52,6 +52,10 @@ class DownloadImagesSubscriber implements EventSubscriber $config = new $this->configClass(); $config->setEntityManager($args->getEntityManager()); + if (!$config->get('download_images_enabled')) { + return; + } + // field content has been updated if ($args->hasChangedField('content')) { $html = $this->downloadImages($config, $entity); @@ -87,6 +91,10 @@ class DownloadImagesSubscriber implements EventSubscriber $config = new $this->configClass(); $config->setEntityManager($args->getEntityManager()); + if (!$config->get('download_images_enabled')) { + return; + } + // update all images inside the html $html = $this->downloadImages($config, $entity); if (false !== $html) { -- cgit v1.2.3 From aedd6ca0fd212abd07ec59c5fd58ea2ca99198c5 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Mon, 31 Oct 2016 13:29:33 +0100 Subject: Add translations & migration --- src/Wallabag/CoreBundle/Command/InstallCommand.php | 14 ++------------ .../CoreBundle/DataFixtures/ORM/LoadSettingData.php | 12 +----------- .../Event/Subscriber/DownloadImagesSubscriber.php | 16 ++++------------ 3 files changed, 7 insertions(+), 35 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Command/InstallCommand.php b/src/Wallabag/CoreBundle/Command/InstallCommand.php index aedccfe4..9fe90357 100644 --- a/src/Wallabag/CoreBundle/Command/InstallCommand.php +++ b/src/Wallabag/CoreBundle/Command/InstallCommand.php @@ -370,7 +370,7 @@ class InstallCommand extends ContainerAwareCommand ], [ 'name' => 'wallabag_url', - 'value' => 'http://v2.wallabag.org', + 'value' => '', 'section' => 'misc', ], [ @@ -401,17 +401,7 @@ class InstallCommand extends ContainerAwareCommand [ 'name' => 'download_images_enabled', 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_rabbitmq', - 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_redis', - 'value' => '0', - 'section' => 'image', + 'section' => 'misc', ], ]; diff --git a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php index 70a7a4ac..d0085660 100644 --- a/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php +++ b/src/Wallabag/CoreBundle/DataFixtures/ORM/LoadSettingData.php @@ -143,17 +143,7 @@ class LoadSettingData extends AbstractFixture implements OrderedFixtureInterface [ 'name' => 'download_images_enabled', 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_rabbitmq', - 'value' => '0', - 'section' => 'image', - ], - [ - 'name' => 'download_images_with_redis', - 'value' => '0', - 'section' => 'image', + 'section' => 'misc', ], ]; diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 3f2d460c..6fddcea9 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -111,6 +111,8 @@ class DownloadImagesSubscriber implements EventSubscriber /** * Download all images from the html. * + * @todo If we want to add async download, it should be done in that method + * * @param Config $config * @param Entry $entry * @@ -120,12 +122,6 @@ class DownloadImagesSubscriber implements EventSubscriber { $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - // if ($config->get('download_images_with_rabbitmq')) { - - // } else if ($config->get('download_images_with_redis')) { - - // } - return $this->downloadImages->processHtml( $entry->getContent(), $entry->getUrl() @@ -135,6 +131,8 @@ class DownloadImagesSubscriber implements EventSubscriber /** * Download the preview picture. * + * @todo If we want to add async download, it should be done in that method + * * @param Config $config * @param Entry $entry * @@ -144,12 +142,6 @@ class DownloadImagesSubscriber implements EventSubscriber { $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - // if ($config->get('download_images_with_rabbitmq')) { - - // } else if ($config->get('download_images_with_redis')) { - - // } - return $this->downloadImages->processSingleImage( $entry->getPreviewPicture(), $entry->getUrl() -- cgit v1.2.3 From e0597476d1d5f6a4a7d6ea9b76966465f3d22fb8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 1 Nov 2016 14:49:02 +0100 Subject: Use custom event instead of Doctrine ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This give us ability to use Entry ID to determine where to store images and it’s then more easy to remove them when we remove the entry. --- .../CoreBundle/Controller/EntryController.php | 14 +++ .../CoreBundle/Event/EntryDeletedEvent.php | 26 +++++ src/Wallabag/CoreBundle/Event/EntrySavedEvent.php | 26 +++++ .../Event/Subscriber/DownloadImagesSubscriber.php | 117 ++++++++------------- src/Wallabag/CoreBundle/Helper/DownloadImages.php | 84 +++++++++------ .../CoreBundle/Resources/config/services.yml | 6 +- 6 files changed, 165 insertions(+), 108 deletions(-) create mode 100644 src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php create mode 100644 src/Wallabag/CoreBundle/Event/EntrySavedEvent.php (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Controller/EntryController.php b/src/Wallabag/CoreBundle/Controller/EntryController.php index 97bb3d12..3f4eb17d 100644 --- a/src/Wallabag/CoreBundle/Controller/EntryController.php +++ b/src/Wallabag/CoreBundle/Controller/EntryController.php @@ -13,6 +13,8 @@ use Wallabag\CoreBundle\Form\Type\EntryFilterType; use Wallabag\CoreBundle\Form\Type\EditEntryType; use Wallabag\CoreBundle\Form\Type\NewEntryType; use Sensio\Bundle\FrameworkExtraBundle\Configuration\Cache; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; class EntryController extends Controller { @@ -81,6 +83,9 @@ class EntryController extends Controller $em->persist($entry); $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + return $this->redirect($this->generateUrl('homepage')); } @@ -107,6 +112,9 @@ class EntryController extends Controller $em = $this->getDoctrine()->getManager(); $em->persist($entry); $em->flush(); + + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); } return $this->redirect($this->generateUrl('homepage')); @@ -343,6 +351,9 @@ class EntryController extends Controller $em->persist($entry); $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + return $this->redirect($this->generateUrl('view', ['id' => $entry->getId()])); } @@ -431,6 +442,9 @@ class EntryController extends Controller UrlGeneratorInterface::ABSOLUTE_PATH ); + // entry deleted, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry)); + $em = $this->getDoctrine()->getManager(); $em->remove($entry); $em->flush(); diff --git a/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php b/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php new file mode 100644 index 00000000..e9061d04 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/EntryDeletedEvent.php @@ -0,0 +1,26 @@ +entry = $entry; + } + + public function getEntry() + { + return $this->entry; + } +} diff --git a/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php b/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php new file mode 100644 index 00000000..5fdb5221 --- /dev/null +++ b/src/Wallabag/CoreBundle/Event/EntrySavedEvent.php @@ -0,0 +1,26 @@ +entry = $entry; + } + + public function getEntry() + { + return $this->entry; + } +} diff --git a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php index 6fddcea9..4ebe837b 100644 --- a/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php +++ b/src/Wallabag/CoreBundle/Event/Subscriber/DownloadImagesSubscriber.php @@ -2,110 +2,85 @@ namespace Wallabag\CoreBundle\Event\Subscriber; -use Doctrine\Common\EventSubscriber; -use Doctrine\ORM\Event\LifecycleEventArgs; +use Symfony\Component\EventDispatcher\EventSubscriberInterface; use Psr\Log\LoggerInterface; use Wallabag\CoreBundle\Helper\DownloadImages; use Wallabag\CoreBundle\Entity\Entry; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; use Doctrine\ORM\EntityManager; -use Craue\ConfigBundle\Util\Config; -class DownloadImagesSubscriber implements EventSubscriber +class DownloadImagesSubscriber implements EventSubscriberInterface { - private $configClass; + private $em; private $downloadImages; + private $enabled; private $logger; - /** - * We inject the class instead of the service otherwise it generates a circular reference with the EntityManager. - * So we build the service ourself when we got the EntityManager (in downloadImages). - */ - public function __construct(DownloadImages $downloadImages, $configClass, LoggerInterface $logger) + public function __construct(EntityManager $em, DownloadImages $downloadImages, $enabled, LoggerInterface $logger) { + $this->em = $em; $this->downloadImages = $downloadImages; - $this->configClass = $configClass; + $this->enabled = $enabled; $this->logger = $logger; } - public function getSubscribedEvents() + public static function getSubscribedEvents() { - return array( - 'prePersist', - 'preUpdate', - ); + return [ + EntrySavedEvent::NAME => 'onEntrySaved', + EntryDeletedEvent::NAME => 'onEntryDeleted', + ]; } /** - * In case of an entry has been updated. - * We won't update the content field if it wasn't updated. + * Download images and updated the data into the entry. * - * @param LifecycleEventArgs $args + * @param EntrySavedEvent $event */ - public function preUpdate(LifecycleEventArgs $args) + public function onEntrySaved(EntrySavedEvent $event) { - $entity = $args->getEntity(); + if (!$this->enabled) { + $this->logger->debug('DownloadImagesSubscriber: disabled.'); - if (!$entity instanceof Entry) { return; } - $config = new $this->configClass(); - $config->setEntityManager($args->getEntityManager()); - - if (!$config->get('download_images_enabled')) { - return; - } + $entry = $event->getEntry(); - // field content has been updated - if ($args->hasChangedField('content')) { - $html = $this->downloadImages($config, $entity); + $html = $this->downloadImages($entry); + if (false !== $html) { + $this->logger->debug('DownloadImagesSubscriber: updated html.'); - if (false !== $html) { - $args->setNewValue('content', $html); - } + $entry->setContent($html); } - // field preview picture has been updated - if ($args->hasChangedField('previewPicture')) { - $previewPicture = $this->downloadPreviewImage($config, $entity); + // update preview picture + $previewPicture = $this->downloadPreviewImage($entry); + if (false !== $previewPicture) { + $this->logger->debug('DownloadImagesSubscriber: update preview picture.'); - if (false !== $previewPicture) { - $entity->setPreviewPicture($previewPicture); - } + $entry->setPreviewPicture($previewPicture); } + + $this->em->persist($entry); + $this->em->flush(); } /** - * When a new entry is saved. + * Remove images related to the entry. * - * @param LifecycleEventArgs $args + * @param EntryDeletedEvent $event */ - public function prePersist(LifecycleEventArgs $args) + public function onEntryDeleted(EntryDeletedEvent $event) { - $entity = $args->getEntity(); - - if (!$entity instanceof Entry) { - return; - } - - $config = new $this->configClass(); - $config->setEntityManager($args->getEntityManager()); + if (!$this->enabled) { + $this->logger->debug('DownloadImagesSubscriber: disabled.'); - if (!$config->get('download_images_enabled')) { return; } - // update all images inside the html - $html = $this->downloadImages($config, $entity); - if (false !== $html) { - $entity->setContent($html); - } - - // update preview picture - $previewPicture = $this->downloadPreviewImage($config, $entity); - if (false !== $previewPicture) { - $entity->setPreviewPicture($previewPicture); - } + $this->downloadImages->removeImages($event->getEntry()->getId()); } /** @@ -113,16 +88,14 @@ class DownloadImagesSubscriber implements EventSubscriber * * @todo If we want to add async download, it should be done in that method * - * @param Config $config - * @param Entry $entry + * @param Entry $entry * * @return string|false False in case of async */ - public function downloadImages(Config $config, Entry $entry) + private function downloadImages(Entry $entry) { - $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - return $this->downloadImages->processHtml( + $entry->getId(), $entry->getContent(), $entry->getUrl() ); @@ -133,16 +106,14 @@ class DownloadImagesSubscriber implements EventSubscriber * * @todo If we want to add async download, it should be done in that method * - * @param Config $config - * @param Entry $entry + * @param Entry $entry * * @return string|false False in case of async */ - public function downloadPreviewImage(Config $config, Entry $entry) + private function downloadPreviewImage(Entry $entry) { - $this->downloadImages->setWallabagUrl($config->get('wallabag_url')); - return $this->downloadImages->processSingleImage( + $entry->getId(), $entry->getPreviewPicture(), $entry->getUrl() ); diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php index e7982c56..c5298236 100644 --- a/src/Wallabag/CoreBundle/Helper/DownloadImages.php +++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php @@ -6,6 +6,7 @@ use Psr\Log\LoggerInterface; use Symfony\Component\DomCrawler\Crawler; use GuzzleHttp\Client; use Symfony\Component\HttpFoundation\File\MimeType\MimeTypeExtensionGuesser; +use Symfony\Component\Finder\Finder; class DownloadImages { @@ -17,27 +18,17 @@ class DownloadImages private $mimeGuesser; private $wallabagUrl; - public function __construct(Client $client, $baseFolder, LoggerInterface $logger) + public function __construct(Client $client, $baseFolder, $wallabagUrl, LoggerInterface $logger) { $this->client = $client; $this->baseFolder = $baseFolder; + $this->wallabagUrl = rtrim($wallabagUrl, '/'); $this->logger = $logger; $this->mimeGuesser = new MimeTypeExtensionGuesser(); $this->setFolder(); } - /** - * Since we can't inject CraueConfig service because it'll generate a circular reference when injected in the subscriber - * we use a different way to inject the current wallabag url. - * - * @param string $url Usually from `$config->get('wallabag_url')` - */ - public function setWallabagUrl($url) - { - $this->wallabagUrl = rtrim($url, '/'); - } - /** * Setup base folder where all images are going to be saved. */ @@ -52,23 +43,24 @@ class DownloadImages /** * Process the html and extract image from it, save them to local and return the updated html. * + * @param int $entryId ID of the entry * @param string $html - * @param string $url Used as a base path for relative image and folder + * @param string $url Used as a base path for relative image and folder * * @return string */ - public function processHtml($html, $url) + public function processHtml($entryId, $html, $url) { $crawler = new Crawler($html); $result = $crawler ->filterXpath('//img') ->extract(array('src')); - $relativePath = $this->getRelativePath($url); + $relativePath = $this->getRelativePath($entryId); // download and save the image to the folder foreach ($result as $image) { - $imagePath = $this->processSingleImage($image, $url, $relativePath); + $imagePath = $this->processSingleImage($entryId, $image, $url, $relativePath); if (false === $imagePath) { continue; @@ -86,24 +78,27 @@ class DownloadImages * - re-saved it (for security reason) * - return the new local path. * + * @param int $entryId ID of the entry * @param string $imagePath Path to the image to retrieve * @param string $url Url from where the image were found * @param string $relativePath Relative local path to saved the image * * @return string Relative url to access the image from the web */ - public function processSingleImage($imagePath, $url, $relativePath = null) + public function processSingleImage($entryId, $imagePath, $url, $relativePath = null) { - if (null == $relativePath) { - $relativePath = $this->getRelativePath($url); + if (null === $relativePath) { + $relativePath = $this->getRelativePath($entryId); } + $this->logger->debug('DownloadImages: working on image: '.$imagePath); + $folderPath = $this->baseFolder.'/'.$relativePath; // build image path $absolutePath = $this->getAbsoluteLink($url, $imagePath); if (false === $absolutePath) { - $this->logger->log('error', 'Can not determine the absolute path for that image, skipping.'); + $this->logger->error('DownloadImages: Can not determine the absolute path for that image, skipping.'); return false; } @@ -111,15 +106,15 @@ class DownloadImages try { $res = $this->client->get($absolutePath); } catch (\Exception $e) { - $this->logger->log('error', 'Can not retrieve image, skipping.', ['exception' => $e]); + $this->logger->error('DownloadImages: Can not retrieve image, skipping.', ['exception' => $e]); return false; } $ext = $this->mimeGuesser->guess($res->getHeader('content-type')); - $this->logger->log('debug', 'Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); + $this->logger->debug('DownloadImages: Checking extension', ['ext' => $ext, 'header' => $res->getHeader('content-type')]); if (!in_array($ext, ['jpeg', 'jpg', 'gif', 'png'], true)) { - $this->logger->log('error', 'Processed image with not allowed extension. Skipping '.$imagePath); + $this->logger->error('DownloadImages: Processed image with not allowed extension. Skipping '.$imagePath); return false; } @@ -133,7 +128,7 @@ class DownloadImages } if (false === $im) { - $this->logger->log('error', 'Error while regenerating image', ['path' => $localPath]); + $this->logger->error('DownloadImages: Error while regenerating image', ['path' => $localPath]); return false; } @@ -141,16 +136,16 @@ class DownloadImages switch ($ext) { case 'gif': $result = imagegif($im, $localPath); - $this->logger->log('debug', 'Re-creating gif'); + $this->logger->debug('DownloadImages: Re-creating gif'); break; case 'jpeg': case 'jpg': $result = imagejpeg($im, $localPath, self::REGENERATE_PICTURES_QUALITY); - $this->logger->log('debug', 'Re-creating jpg'); + $this->logger->debug('DownloadImages: Re-creating jpg'); break; case 'png': $result = imagepng($im, $localPath, ceil(self::REGENERATE_PICTURES_QUALITY / 100 * 9)); - $this->logger->log('debug', 'Re-creating png'); + $this->logger->debug('DownloadImages: Re-creating png'); } imagedestroy($im); @@ -158,24 +153,47 @@ class DownloadImages return $this->wallabagUrl.'/assets/images/'.$relativePath.'/'.$hashImage.'.'.$ext; } + /** + * Remove all images for the given entry id. + * + * @param int $entryId ID of the entry + */ + public function removeImages($entryId) + { + $relativePath = $this->getRelativePath($entryId); + $folderPath = $this->baseFolder.'/'.$relativePath; + + $finder = new Finder(); + $finder + ->files() + ->ignoreDotFiles(true) + ->in($folderPath); + + foreach ($finder as $file) { + @unlink($file->getRealPath()); + } + + @rmdir($folderPath); + } + /** * Generate the folder where we are going to save images based on the entry url. * - * @param string $url + * @param int $entryId ID of the entry * * @return string */ - private function getRelativePath($url) + private function getRelativePath($entryId) { - $hashUrl = hash('crc32', $url); - $relativePath = $hashUrl[0].'/'.$hashUrl[1].'/'.$hashUrl; + $hashId = hash('crc32', $entryId); + $relativePath = $hashId[0].'/'.$hashId[1].'/'.$hashId; $folderPath = $this->baseFolder.'/'.$relativePath; if (!file_exists($folderPath)) { mkdir($folderPath, 0777, true); } - $this->logger->log('debug', 'Folder used for that url', ['folder' => $folderPath, 'url' => $url]); + $this->logger->debug('DownloadImages: Folder used for that Entry id', ['folder' => $folderPath, 'entryId' => $entryId]); return $relativePath; } @@ -208,7 +226,7 @@ class DownloadImages return $absolute->get_uri(); } - $this->logger->log('error', 'Can not make an absolute link', ['base' => $base, 'url' => $url]); + $this->logger->error('DownloadImages: Can not make an absolute link', ['base' => $base, 'url' => $url]); return false; } diff --git a/src/Wallabag/CoreBundle/Resources/config/services.yml b/src/Wallabag/CoreBundle/Resources/config/services.yml index 1fb81a46..56d776ad 100644 --- a/src/Wallabag/CoreBundle/Resources/config/services.yml +++ b/src/Wallabag/CoreBundle/Resources/config/services.yml @@ -140,17 +140,19 @@ services: wallabag_core.subscriber.download_images: class: Wallabag\CoreBundle\Event\Subscriber\DownloadImagesSubscriber arguments: + - "@doctrine.orm.default_entity_manager" - "@wallabag_core.entry.download_images" - - "%craue_config.config.class%" + - '@=service(''craue_config'').get(''download_images_enabled'')' - "@logger" tags: - - { name: doctrine.event_subscriber } + - { name: kernel.event_subscriber } wallabag_core.entry.download_images: class: Wallabag\CoreBundle\Helper\DownloadImages arguments: - "@wallabag_core.entry.download_images.client" - "%kernel.root_dir%/../web/assets/images" + - '@=service(''craue_config'').get(''wallabag_url'')' - "@logger" wallabag_core.entry.download_images.client: -- cgit v1.2.3 From 7816eb622df2353cea0ede0a3674d5eb3a01a1a9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:10:23 +0100 Subject: Add entry.saved event to import & rest --- .../Controller/WallabagRestController.php | 9 +++++++- .../ImportBundle/Consumer/AbstractConsumer.php | 8 +++++++- .../ImportBundle/Import/AbstractImport.php | 24 +++++++++++++++++++++- src/Wallabag/ImportBundle/Import/BrowserImport.php | 17 +++++++++++++++ src/Wallabag/ImportBundle/Import/PocketImport.php | 7 ------- .../ImportBundle/Resources/config/rabbit.yml | 7 +++++++ .../ImportBundle/Resources/config/redis.yml | 7 +++++++ .../ImportBundle/Resources/config/services.yml | 7 +++++++ 8 files changed, 76 insertions(+), 10 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/ApiBundle/Controller/WallabagRestController.php b/src/Wallabag/ApiBundle/Controller/WallabagRestController.php index a73d44ca..50652b77 100644 --- a/src/Wallabag/ApiBundle/Controller/WallabagRestController.php +++ b/src/Wallabag/ApiBundle/Controller/WallabagRestController.php @@ -14,6 +14,8 @@ use Symfony\Component\Security\Core\Exception\AccessDeniedException; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\AnnotationBundle\Entity\Annotation; +use Wallabag\CoreBundle\Event\EntrySavedEvent; +use Wallabag\CoreBundle\Event\EntryDeletedEvent; class WallabagRestController extends FOSRestController { @@ -233,9 +235,11 @@ class WallabagRestController extends FOSRestController $em = $this->getDoctrine()->getManager(); $em->persist($entry); - $em->flush(); + // entry saved, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + $json = $this->get('serializer')->serialize($entry, 'json'); return (new JsonResponse())->setJson($json); @@ -308,6 +312,9 @@ class WallabagRestController extends FOSRestController $this->validateAuthentication(); $this->validateUserAccess($entry->getUser()->getId()); + // entry deleted, dispatch event about it! + $this->get('event_dispatcher')->dispatch(EntryDeletedEvent::NAME, new EntryDeletedEvent($entry)); + $em = $this->getDoctrine()->getManager(); $em->remove($entry); $em->flush(); diff --git a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php index b893ea29..aa7ff914 100644 --- a/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php +++ b/src/Wallabag/ImportBundle/Consumer/AbstractConsumer.php @@ -9,6 +9,8 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractConsumer { @@ -17,11 +19,12 @@ abstract class AbstractConsumer protected $import; protected $logger; - public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, LoggerInterface $logger = null) + public function __construct(EntityManager $em, UserRepository $userRepository, AbstractImport $import, EventDispatcherInterface $eventDispatcher, LoggerInterface $logger = null) { $this->em = $em; $this->userRepository = $userRepository; $this->import = $import; + $this->eventDispatcher = $eventDispatcher; $this->logger = $logger ?: new NullLogger(); } @@ -59,6 +62,9 @@ abstract class AbstractConsumer try { $this->em->flush(); + // entry saved, dispatch event about it! + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 764b390a..1d4a6e27 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php @@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\CoreBundle\Entity\Tag; use Wallabag\UserBundle\Entity\User; use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; +use Symfony\Component\EventDispatcher\EventDispatcherInterface; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class AbstractImport implements ImportInterface { protected $em; protected $logger; protected $contentProxy; + protected $eventDispatcher; protected $producer; protected $user; protected $markAsRead; @@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface protected $importedEntries = 0; protected $queuedEntries = 0; - public function __construct(EntityManager $em, ContentProxy $contentProxy) + public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher) { $this->em = $em; $this->logger = new NullLogger(); $this->contentProxy = $contentProxy; + $this->eventDispatcher = $eventDispatcher; } public function setLogger(LoggerInterface $logger) @@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ($this->markAsRead) { @@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface continue; } + // store each entry to be flushed so we can trigger the entry.saved event for each of them + // entry.saved needs the entry to be persisted in db because it needs it id to generate + // images (at least) + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; + // clear only affected entities $this->em->clear(Entry::class); $this->em->clear(Tag::class); @@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 2ca1683b..8bf7d92e 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import; use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; +use Wallabag\CoreBundle\Event\EntrySavedEvent; abstract class BrowserImport extends AbstractImport { @@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport protected function parseEntries($entries) { $i = 1; + $entryToBeFlushed = []; foreach ($entries as $importedEntry) { if ((array) $importedEntry !== $importedEntry) { @@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport continue; } + // @see AbstractImport + $entryToBeFlushed[] = $entry; + // flush every 20 entries if (($i % 20) === 0) { $this->em->flush(); + + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + + $entryToBeFlushed = []; } ++$i; } $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); + } + } } /** diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 327e2500..0c26aced 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -16,13 +16,6 @@ class PocketImport extends AbstractImport const NB_ELEMENTS = 5000; - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->contentProxy = $contentProxy; - $this->logger = new NullLogger(); - } - /** * Only used for test purpose. * diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index 70b8a0d4..a5af5282 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -6,6 +6,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.readability: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -13,6 +14,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.instapaper: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -20,6 +22,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.instapaper.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -27,6 +30,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.wallabag_v2: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -34,6 +38,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.firefox: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -41,6 +46,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.firefox.import" + - "@event_dispatcher" - "@logger" wallabag_import.consumer.amqp.chrome: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -48,4 +54,5 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.chrome.import" + - "@event_dispatcher" - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 0a81e1b5..5ced4c83 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -18,6 +18,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.readability.import" + - "@event_dispatcher" - "@logger" # instapaper @@ -38,6 +39,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.instapaper.import" + - "@event_dispatcher" - "@logger" # pocket @@ -58,6 +60,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.pocket.import" + - "@event_dispatcher" - "@logger" # wallabag v1 @@ -78,6 +81,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v1.import" + - "@event_dispatcher" - "@logger" # wallabag v2 @@ -98,6 +102,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" + - "@event_dispatcher" - "@logger" # firefox @@ -118,6 +123,7 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.firefox.import" + - "@event_dispatcher" - "@logger" # chrome @@ -138,4 +144,5 @@ services: - "@doctrine.orm.entity_manager" - "@wallabag_user.user_repository" - "@wallabag_import.chrome.import" + - "@event_dispatcher" - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index d600be0f..64822963 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -20,6 +20,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setClient, [ "@wallabag_import.pocket.client" ] ] - [ setLogger, [ "@logger" ]] @@ -31,6 +32,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -41,6 +43,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -51,6 +54,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -61,6 +65,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -71,6 +76,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: @@ -80,6 +86,7 @@ services: arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" + - "@event_dispatcher" calls: - [ setLogger, [ "@logger" ]] tags: -- cgit v1.2.3 From 1f66d79e6b5f54375e63799f10d0773203fe12f9 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:10:57 +0100 Subject: Add more importer to wallabag:import command All importer available expect Pocket which require an oAuth login. --- .../ImportBundle/Command/ImportCommand.php | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 1df38295..2f7a906e 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -14,10 +14,10 @@ class ImportCommand extends ContainerAwareCommand { $this ->setName('wallabag:import') - ->setDescription('Import entries from a JSON export from a wallabag v1 instance') + ->setDescription('Import entries from a JSON export') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, readability, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -42,29 +42,35 @@ class ImportCommand extends ContainerAwareCommand switch ($input->getOption('importer')) { case 'v2': - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); + $import = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); break; case 'firefox': - $wallabag = $this->getContainer()->get('wallabag_import.firefox.import'); + $import = $this->getContainer()->get('wallabag_import.firefox.import'); break; case 'chrome': - $wallabag = $this->getContainer()->get('wallabag_import.chrome.import'); + $import = $this->getContainer()->get('wallabag_import.chrome.import'); + break; + case 'readability': + $import = $this->getContainer()->get('wallabag_import.readability.import'); + break; + case 'instapaper': + $import = $this->getContainer()->get('wallabag_import.instapaper.import'); break; case 'v1': default: - $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); + $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); break; } - $wallabag->setMarkAsRead($input->getOption('markAsRead')); - $wallabag->setUser($user); + $import->setMarkAsRead($input->getOption('markAsRead')); + $import->setUser($user); - $res = $wallabag + $res = $import ->setFilepath($input->getArgument('filepath')) ->import(); if (true === $res) { - $summary = $wallabag->getSummary(); + $summary = $import->getSummary(); $output->writeln(''.$summary['imported'].' imported'); $output->writeln(''.$summary['skipped'].' already saved'); } -- cgit v1.2.3 From e46fdf3096d55028611713b7a196401aa57d864b Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Wed, 2 Nov 2016 07:26:14 +0100 Subject: CS --- src/Wallabag/ImportBundle/Import/PocketImport.php | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/Wallabag') diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 0c26aced..33093480 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php @@ -2,8 +2,6 @@ namespace Wallabag\ImportBundle\Import; -use Psr\Log\NullLogger; -use Doctrine\ORM\EntityManager; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use Wallabag\CoreBundle\Entity\Entry; -- cgit v1.2.3 From ca08d02f2f17c0e4ec0152a701c540f62713a428 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Thu, 3 Nov 2016 15:59:18 +0100 Subject: Add warning message for import + download images When import isn't async and downloading images is enabled, we warn the user that import can fail. Also, added missing translations --- .../CoreBundle/Resources/translations/messages.da.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.de.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.en.yml | 2 ++ .../CoreBundle/Resources/translations/messages.es.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.fa.yml | 7 +++++++ .../CoreBundle/Resources/translations/messages.fr.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.it.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.oc.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.pl.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.ro.yml | 6 ++++++ .../CoreBundle/Resources/translations/messages.tr.yml | 7 +++++++ .../ImportBundle/Resources/views/Chrome/index.html.twig | 2 ++ .../ImportBundle/Resources/views/Firefox/index.html.twig | 2 ++ .../Resources/views/Import/_information.html.twig | 15 +++++++++++++++ .../Resources/views/Import/_workerEnabled.html.twig | 8 -------- .../ImportBundle/Resources/views/Import/index.html.twig | 2 ++ .../Resources/views/Instapaper/index.html.twig | 2 +- .../ImportBundle/Resources/views/Pocket/index.html.twig | 2 +- .../Resources/views/Readability/index.html.twig | 2 +- .../Resources/views/WallabagV1/index.html.twig | 2 +- 20 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig delete mode 100644 src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig (limited to 'src/Wallabag') diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index c0595583..87083799 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." # firefox: # page_title: 'Import > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index 0051da2f..5e5dd996 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -367,6 +367,7 @@ import: how_to: 'Bitte wähle deinen Readability Export aus und klicke den unteren Button für das Hochladen und Importieren dessen.' worker: enabled: "Der Import erfolgt asynchron. Sobald der Import gestartet ist, wird diese Aufgabe extern abgearbeitet. Der aktuelle Service dafür ist:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Aus Firefox importieren' description: "Dieser Import wird all deine Lesezeichen aus Firefox importieren. Gehe zu deinen Lesezeichen (Strg+Shift+O), dann auf \"Importen und Sichern\", wähle \"Sichern…\". Du erhälst eine .json Datei." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Neuer Client erstellt.' client_deleted: 'Client gelöscht' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 462be556..ab56cdb9 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -367,6 +367,7 @@ import: how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Import > Firefox' description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -470,6 +471,7 @@ flashes: rss_updated: 'RSS information updated' tagging_rules_updated: 'Tagging rules updated' tagging_rules_deleted: 'Tagging rule deleted' + # user_added: 'User "%username%" added' rss_token_updated: 'RSS token updated' annotations_reset: Annotations reset tags_reset: Tags reset diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index cfabe09f..c852e5b7 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nuevo cliente creado.' client_deleted: 'Cliente suprimido' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 07b5bee7..cf072450 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -284,6 +284,7 @@ quickstart: paragraph_2: 'ادامه دهید!' configure: title: 'برنامه را تنظیم کنید' + # description: 'In order to have an application which suits you, have a look into the configuration of wallabag.' language: 'زبان و نمای برنامه را تغییر دهید' rss: 'خوراک آر-اس-اس را فعال کنید' tagging_rules: 'قانون‌های برچسب‌گذاری خودکار مقاله‌هایتان را تعریف کنید' @@ -366,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'درون‌ریزی > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -503,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index db6f9f7e..c4aa3751 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -367,6 +367,7 @@ import: how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer." worker: enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :" + download_images_warning: "Vous avez configuré le téléchagement des images pour vos articles. Combiné à l'import classique, cette opération peut être très très longue (voire échouer). Nous vous conseillons vivement d'activer les imports asynchrones." firefox: page_title: 'Import > Firefox' description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox. Ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json.

" @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nouveau client %name% créé' client_deleted: 'Client %name% supprimé' + user: + notice: + added: 'Utilisateur "%username%" ajouté' + updated: 'Utilisateur "%username%" mis à jour' + deleted: 'Utilisateur "%username%" supprimé' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index f1aff51a..38c780dd 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importa da > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nuovo client creato.' client_deleted: 'Client eliminato' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index e0567d7e..103ca831 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -367,6 +367,7 @@ import: how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." worker: enabled: "L'importacion se fa de manièra asincròna. Un còp l'importacion lançada, una aisina externa s'ocuparà dels messatges un per un. Lo servici actual es : " + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' description: "Aquesta aisina importarà totas vòstres favorits de Firefox. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Novèl client creat' client_deleted: 'Client suprimit' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index a2989dbd..1680dae0 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -367,6 +367,7 @@ import: how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' worker: enabled: "Import jest wykonywany asynchronicznie. Od momentu rozpoczęcia importu, zewnętrzna usługa może zajmować się na raz tylko jednym zadaniem. Bieżącą usługą jest:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Import > Firefox' description: "Ten importer zaimportuje wszystkie twoje zakładki z Firefoksa. Idź do twoich zakładek (Ctrl+Shift+O), następnie w \"Import i kopie zapasowe\", wybierz \"Utwórz kopię zapasową...\". Uzyskasz plik .json." @@ -504,3 +505,8 @@ flashes: notice: client_created: 'Nowy klient utworzony.' client_deleted: 'Klient usunięty' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 6e4813e5..16d39188 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -367,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." # firefox: # page_title: 'Import > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -504,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 76903102..c9a269ec 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -108,6 +108,7 @@ config: # if_label: 'if' # then_tag_as_label: 'then tag as' # delete_rule_label: 'delete' + # edit_rule_label: 'edit' rule_label: 'Kural' tags_label: 'Etiketler' faq: @@ -366,6 +367,7 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'İçe Aktar > Firefox' # description: "This importer will import all your Firefox bookmarks. Just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." @@ -503,3 +505,8 @@ flashes: notice: # client_created: 'New client created.' # client_deleted: 'Client deleted' + user: + notice: + # added: 'User "%username%" added' + # updated: 'User "%username%" updated' + # deleted: 'User "%username%" deleted' diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig index ead828c6..93b08540 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} +
{{ import.description|trans|raw }}

{{ 'import.chrome.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig index f975da3f..ced3f008 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} +
{{ import.description|trans|raw }}

{{ 'import.firefox.how_to'|trans }}

diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig new file mode 100644 index 00000000..48bbcfe7 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Import/_information.html.twig @@ -0,0 +1,15 @@ +{% set redis = craue_setting('import_with_redis') %} +{% set rabbit = craue_setting('import_with_rabbitmq') %} +{% set downloadImages = craue_setting('download_images_enabled') %} + +{% if redis or rabbit %} +
+ {{ 'import.worker.enabled'|trans }} {% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %} +
+{% endif %} + +{% if not redis and not rabbit and downloadImages %} +
+ {{ 'import.worker.download_images_warning'|trans|raw }} +
+{% endif %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig deleted file mode 100644 index 2390a41f..00000000 --- a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig +++ /dev/null @@ -1,8 +0,0 @@ -{% set redis = craue_setting('import_with_redis') %} -{% set rabbit = craue_setting('import_with_rabbitmq') %} - -{% if redis or rabbit %} -
- {{ 'import.worker.enabled'|trans }} {% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %} -
-{% endif %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig index 6ea5e0f4..b1ec40a6 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Import/index.html.twig @@ -6,6 +6,8 @@
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} + {{ 'import.page_description'|trans }}
    {% for import in imports %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig index 5789361f..28165d19 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Instapaper/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig index 6195fa07..536e3d1a 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %} {% if not has_consumer_key %}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig index 74653b0f..737b0adf 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    diff --git a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig index 0b19bc34..974b2c73 100644 --- a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig @@ -6,7 +6,7 @@
    - {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %} + {% include 'WallabagImportBundle:Import:_information.html.twig' %}
    {{ import.description|trans }}
    -- cgit v1.2.3