aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorThomas Citharel <tcit@tcit.fr>2016-06-28 19:07:55 +0200
committerJeremy Benoist <jeremy.benoist@gmail.com>2016-10-29 16:25:45 +0200
commit419214d7221e0821ef2b73eb2b3db816ed0cf173 (patch)
treed64c8af2929d2a6d842895096c615813efd84fcb
parent4dface66707688ea440a6a7569795a85631d1ff0 (diff)
downloadwallabag-419214d7221e0821ef2b73eb2b3db816ed0cf173.tar.gz
wallabag-419214d7221e0821ef2b73eb2b3db816ed0cf173.tar.zst
wallabag-419214d7221e0821ef2b73eb2b3db816ed0cf173.zip
Download pictures successfully
Needs to rewrite them properly (get base url)
-rw-r--r--composer.json3
-rw-r--r--src/Wallabag/CoreBundle/Helper/ContentProxy.php178
-rw-r--r--src/Wallabag/CoreBundle/Helper/DownloadImages.php147
3 files changed, 158 insertions, 170 deletions
diff --git a/composer.json b/composer.json
index ebc0a7dc..e7b30fa1 100644
--- a/composer.json
+++ b/composer.json
@@ -82,7 +82,8 @@
82 "white-october/pagerfanta-bundle": "^1.0", 82 "white-october/pagerfanta-bundle": "^1.0",
83 "php-amqplib/rabbitmq-bundle": "^1.8", 83 "php-amqplib/rabbitmq-bundle": "^1.8",
84 "predis/predis": "^1.0", 84 "predis/predis": "^1.0",
85 "javibravo/simpleue": "^1.0" 85 "javibravo/simpleue": "^1.0",
86 "symfony/dom-crawler": "^3.1"
86 }, 87 },
87 "require-dev": { 88 "require-dev": {
88 "doctrine/doctrine-fixtures-bundle": "~2.2", 89 "doctrine/doctrine-fixtures-bundle": "~2.2",
diff --git a/src/Wallabag/CoreBundle/Helper/ContentProxy.php b/src/Wallabag/CoreBundle/Helper/ContentProxy.php
index ddeffa77..bbad705f 100644
--- a/src/Wallabag/CoreBundle/Helper/ContentProxy.php
+++ b/src/Wallabag/CoreBundle/Helper/ContentProxy.php
@@ -65,7 +65,7 @@ class ContentProxy
65 65
66 $entry->setUrl($content['url'] ?: $url); 66 $entry->setUrl($content['url'] ?: $url);
67 $entry->setTitle($title); 67 $entry->setTitle($title);
68 $entry->setContent($html); 68
69 $entry->setLanguage($content['language']); 69 $entry->setLanguage($content['language']);
70 $entry->setMimetype($content['content_type']); 70 $entry->setMimetype($content['content_type']);
71 $entry->setReadingTime(Utils::getReadingTime($html)); 71 $entry->setReadingTime(Utils::getReadingTime($html));
@@ -75,6 +75,14 @@ class ContentProxy
75 $entry->setDomainName($domainName); 75 $entry->setDomainName($domainName);
76 } 76 }
77 77
78 if (true) {
79 $this->logger->log('debug','Starting to download images');
80 $downloadImages = new DownloadImages($html, $url, $this->logger);
81 $html = $downloadImages->process();
82 }
83
84 $entry->setContent($html);
85
78 if (isset($content['open_graph']['og_image'])) { 86 if (isset($content['open_graph']['og_image'])) {
79 $entry->setPreviewPicture($content['open_graph']['og_image']); 87 $entry->setPreviewPicture($content['open_graph']['og_image']);
80 } 88 }
@@ -151,172 +159,4 @@ class ContentProxy
151 { 159 {
152 return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']); 160 return isset($content['title']) && isset($content['html']) && isset($content['url']) && isset($content['language']) && isset($content['content_type']);
153 } 161 }
154
155 /**
156 * Changing pictures URL in article content.
157 */
158 public static function filterPicture($content, $url, $id)
159 {
160 $matches = array();
161 $processing_pictures = array(); // list of processing image to avoid processing the same pictures twice
162 preg_match_all('#<\s*(img)[^>]+src="([^"]*)"[^>]*>#Si', $content, $matches, PREG_SET_ORDER);
163 foreach ($matches as $i => $link) {
164 $link[1] = trim($link[1]);
165 if (!preg_match('#^(([a-z]+://)|(\#))#', $link[1])) {
166 $absolute_path = self::_getAbsoluteLink($link[2], $url);
167 $filename = basename(parse_url($absolute_path, PHP_URL_PATH));
168 $directory = self::_createAssetsDirectory($id);
169 $fullpath = $directory.'/'.$filename;
170
171 if (in_array($absolute_path, $processing_pictures) === true) {
172 // replace picture's URL only if processing is OK : already processing -> go to next picture
173 continue;
174 }
175
176 if (self::_downloadPictures($absolute_path, $fullpath) === true) {
177 $content = str_replace($matches[$i][2], Tools::getPocheUrl().$fullpath, $content);
178 }
179
180 $processing_pictures[] = $absolute_path;
181 }
182 }
183
184 return $content;
185 }
186
187 /**
188 * Get absolute URL.
189 */
190 private static function _getAbsoluteLink($relativeLink, $url)
191 {
192 /* return if already absolute URL */
193 if (parse_url($relativeLink, PHP_URL_SCHEME) != '') {
194 return $relativeLink;
195 }
196
197 /* queries and anchors */
198 if ($relativeLink[0] == '#' || $relativeLink[0] == '?') {
199 return $url.$relativeLink;
200 }
201
202 /* parse base URL and convert to local variables:
203 $scheme, $host, $path */
204 extract(parse_url($url));
205
206 /* remove non-directory element from path */
207 $path = preg_replace('#/[^/]*$#', '', $path);
208
209 /* destroy path if relative url points to root */
210 if ($relativeLink[0] == '/') {
211 $path = '';
212 }
213
214 /* dirty absolute URL */
215 $abs = $host.$path.'/'.$relativeLink;
216
217 /* replace '//' or '/./' or '/foo/../' with '/' */
218 $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#');
219 for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) {
220 }
221
222 /* absolute URL is ready! */
223 return $scheme.'://'.$abs;
224 }
225
226 /**
227 * Downloading pictures.
228 *
229 * @return bool true if the download and processing is OK, false else
230 */
231 private static function _downloadPictures($absolute_path, $fullpath)
232 {
233 $rawdata = Tools::getFile($absolute_path);
234 $fullpath = urldecode($fullpath);
235
236 if (file_exists($fullpath)) {
237 unlink($fullpath);
238 }
239
240 // check extension
241 $file_ext = strrchr($fullpath, '.');
242 $whitelist = array('.jpg', '.jpeg', '.gif', '.png');
243 if (!(in_array($file_ext, $whitelist))) {
244 Tools::logm('processed image with not allowed extension. Skipping '.$fullpath);
245
246 return false;
247 }
248
249 // check headers
250 $imageinfo = getimagesize($absolute_path);
251 if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') {
252 Tools::logm('processed image with bad header. Skipping '.$fullpath);
253
254 return false;
255 }
256
257 // regenerate image
258 $im = imagecreatefromstring($rawdata);
259 if ($im === false) {
260 Tools::logm('error while regenerating image '.$fullpath);
261
262 return false;
263 }
264
265 switch ($imageinfo['mime']) {
266 case 'image/gif':
267 $result = imagegif($im, $fullpath);
268 break;
269 case 'image/jpeg':
270 case 'image/jpg':
271 $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY);
272 break;
273 case 'image/png':
274 $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9));
275 break;
276 }
277 imagedestroy($im);
278
279 return $result;
280 }
281
282 /**
283 * Create a directory for an article.
284 *
285 * @param $id ID of the article
286 *
287 * @return string
288 */
289 private static function _createAssetsDirectory($id)
290 {
291 $assets_path = ABS_PATH;
292 if (!is_dir($assets_path)) {
293 mkdir($assets_path, 0715);
294 }
295
296 $article_directory = $assets_path.$id;
297 if (!is_dir($article_directory)) {
298 mkdir($article_directory, 0715);
299 }
300
301 return $article_directory;
302 }
303
304 /**
305 * Remove the directory.
306 *
307 * @param $directory
308 *
309 * @return bool
310 */
311 public static function removeDirectory($directory)
312 {
313 if (is_dir($directory)) {
314 $files = array_diff(scandir($directory), array('.', '..'));
315 foreach ($files as $file) {
316 (is_dir("$directory/$file")) ? self::removeDirectory("$directory/$file") : unlink("$directory/$file");
317 }
318
319 return rmdir($directory);
320 }
321 }
322} 162}
diff --git a/src/Wallabag/CoreBundle/Helper/DownloadImages.php b/src/Wallabag/CoreBundle/Helper/DownloadImages.php
new file mode 100644
index 00000000..32a9dbb2
--- /dev/null
+++ b/src/Wallabag/CoreBundle/Helper/DownloadImages.php
@@ -0,0 +1,147 @@
1<?php
2
3namespace Wallabag\CoreBundle\Helper;
4
5use Psr\Log\LoggerInterface as Logger;
6use Symfony\Component\DomCrawler\Crawler;
7
8define('REGENERATE_PICTURES_QUALITY', 75);
9
10class DownloadImages {
11 private $folder;
12 private $url;
13 private $html;
14 private $fileName;
15 private $logger;
16
17 public function __construct($html, $url, Logger $logger) {
18 $this->html = $html;
19 $this->url = $url;
20 $this->setFolder();
21 $this->logger = $logger;
22 }
23
24 public function setFolder($folder = "assets/images") {
25 // if folder doesn't exist, attempt to create one and store the folder name in property $folder
26 if(!file_exists($folder)) {
27 mkdir($folder);
28 }
29 $this->folder = $folder;
30 }
31
32 public function process() {
33 //instantiate the symfony DomCrawler Component
34 $crawler = new Crawler($this->html);
35 // create an array of all scrapped image links
36 $this->logger->log('debug', 'Finding images inside document');
37 $result = $crawler
38 ->filterXpath('//img')
39 ->extract(array('src'));
40
41 // download and save the image to the folder
42 foreach ($result as $image) {
43 $file = file_get_contents($image);
44
45 // Checks
46 $absolute_path = self::getAbsoluteLink($image, $this->url);
47 $filename = basename(parse_url($absolute_path, PHP_URL_PATH));
48 $fullpath = $this->folder."/".$filename;
49 self::checks($file, $fullpath, $absolute_path);
50 $this->html = str_replace($image, $fullpath, $this->html);
51 }
52
53 return $this->html;
54 }
55
56 private function checks($rawdata, $fullpath, $absolute_path) {
57 $fullpath = urldecode($fullpath);
58
59 if (file_exists($fullpath)) {
60 unlink($fullpath);
61 }
62
63 // check extension
64 $this->logger->log('debug','Checking extension');
65
66 $file_ext = strrchr($fullpath, '.');
67 $whitelist = array('.jpg', '.jpeg', '.gif', '.png');
68 if (!(in_array($file_ext, $whitelist))) {
69 $this->logger->log('debug','processed image with not allowed extension. Skipping '.$fullpath);
70
71 return false;
72 }
73
74 // check headers
75 $this->logger->log('debug','Checking headers');
76 $imageinfo = getimagesize($absolute_path);
77 if ($imageinfo['mime'] != 'image/gif' && $imageinfo['mime'] != 'image/jpeg' && $imageinfo['mime'] != 'image/jpg' && $imageinfo['mime'] != 'image/png') {
78 $this->logger->log('debug','processed image with bad header. Skipping '.$fullpath);
79
80 return false;
81 }
82
83 // regenerate image
84 $this->logger->log('debug','regenerating image');
85 $im = imagecreatefromstring($rawdata);
86 if ($im === false) {
87 $this->logger->log('error','error while regenerating image '.$fullpath);
88
89 return false;
90 }
91
92 switch ($imageinfo['mime']) {
93 case 'image/gif':
94 $result = imagegif($im, $fullpath);
95 $this->logger->log('debug','Re-creating gif');
96 break;
97 case 'image/jpeg':
98 case 'image/jpg':
99 $result = imagejpeg($im, $fullpath, REGENERATE_PICTURES_QUALITY);
100 $this->logger->log('debug','Re-creating jpg');
101 break;
102 case 'image/png':
103 $this->logger->log('debug','Re-creating png');
104 $result = imagepng($im, $fullpath, ceil(REGENERATE_PICTURES_QUALITY / 100 * 9));
105 break;
106 }
107 imagedestroy($im);
108
109 return $result;
110 }
111
112 private static function getAbsoluteLink($relativeLink, $url)
113 {
114 /* return if already absolute URL */
115 if (parse_url($relativeLink, PHP_URL_SCHEME) != '') {
116 return $relativeLink;
117 }
118
119 /* queries and anchors */
120 if ($relativeLink[0] == '#' || $relativeLink[0] == '?') {
121 return $url.$relativeLink;
122 }
123
124 /* parse base URL and convert to local variables:
125 $scheme, $host, $path */
126 extract(parse_url($url));
127
128 /* remove non-directory element from path */
129 $path = preg_replace('#/[^/]*$#', '', $path);
130
131 /* destroy path if relative url points to root */
132 if ($relativeLink[0] == '/') {
133 $path = '';
134 }
135
136 /* dirty absolute URL */
137 $abs = $host.$path.'/'.$relativeLink;
138
139 /* replace '//' or '/./' or '/foo/../' with '/' */
140 $re = array('#(/\.?/)#', '#/(?!\.\.)[^/]+/\.\./#');
141 for ($n = 1; $n > 0; $abs = preg_replace($re, '/', $abs, -1, $n)) {
142 }
143
144 /* absolute URL is ready! */
145 return $scheme.'://'.$abs;
146 }
147}