]>
Commit | Line | Data |
---|---|---|
ec397236 NL |
1 | <?php |
2 | /** | |
3 | * Humble HTTP Agent | |
4 | * | |
5 | * This class is designed to take advantage of parallel HTTP requests | |
6 | * offered by PHP's PECL HTTP extension or the curl_multi_* functions. | |
7 | * For environments which do not have these options, it reverts to standard sequential | |
8 | * requests (using file_get_contents()) | |
9 | * | |
10 | * @version 1.0 | |
11 | * @date 2012-02-09 | |
12 | * @see http://php.net/HttpRequestPool | |
13 | * @author Keyvan Minoukadeh | |
14 | * @copyright 2011-2012 Keyvan Minoukadeh | |
15 | * @license http://www.gnu.org/licenses/agpl-3.0.html AGPL v3 | |
16 | */ | |
17 | ||
18 | class HumbleHttpAgent | |
19 | { | |
20 | const METHOD_REQUEST_POOL = 1; | |
21 | const METHOD_CURL_MULTI = 2; | |
22 | const METHOD_FILE_GET_CONTENTS = 4; | |
23 | //const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'; | |
24 | const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2'; | |
25 | const UA_PHP = 'PHP/5.2'; | |
26 | const REF_GOOGLE = 'http://www.google.co.uk/url?sa=t&source=web&cd=1'; | |
27 | ||
28 | protected $requests = array(); | |
29 | protected $redirectQueue = array(); | |
30 | protected $requestOptions; | |
31 | protected $maxParallelRequests = 5; | |
32 | protected $cache = null; //TODO | |
33 | protected $httpContext; | |
34 | protected $minimiseMemoryUse = false; //TODO | |
35 | protected $debug = false; | |
36 | protected $method; | |
37 | protected $cookieJar; | |
38 | public $rewriteHashbangFragment = true; // see http://code.google.com/web/ajaxcrawling/docs/specification.html | |
39 | public $maxRedirects = 5; | |
40 | public $userAgentMap = array(); | |
41 | public $rewriteUrls = array(); | |
42 | public $userAgentDefault; | |
43 | public $referer; | |
44 | //public $userAgent = 'Mozilla/5.0'; | |
45 | ||
46 | // Prevent certain file/mime types | |
47 | // HTTP responses which match these content types will | |
48 | // be returned without body. | |
49 | public $headerOnlyTypes = array(); | |
50 | // URLs ending with one of these extensions will | |
51 | // prompt Humble HTTP Agent to send a HEAD request first | |
52 | // to see if returned content type matches $headerOnlyTypes. | |
53 | public $headerOnlyClues = array('pdf','mp3','zip','exe','gif','gzip','gz','jpeg','jpg','mpg','mpeg','png','ppt','mov'); | |
54 | ||
55 | //TODO: set max file size | |
56 | //TODO: normalise headers | |
57 | ||
58 | function __construct($requestOptions=null, $method=null) { | |
59 | $this->userAgentDefault = self::UA_BROWSER; | |
60 | $this->referer = self::REF_GOOGLE; | |
61 | // set the request method | |
62 | if (in_array($method, array(1,2,4))) { | |
63 | $this->method = $method; | |
64 | } else { | |
65 | if (class_exists('HttpRequestPool')) { | |
66 | $this->method = self::METHOD_REQUEST_POOL; | |
67 | } elseif (function_exists('curl_multi_init')) { | |
68 | $this->method = self::METHOD_CURL_MULTI; | |
69 | } else { | |
70 | $this->method = self::METHOD_FILE_GET_CONTENTS; | |
71 | } | |
72 | } | |
73 | if ($this->method == self::METHOD_CURL_MULTI) { | |
74 | require_once(dirname(__FILE__).'/RollingCurl.php'); | |
75 | } | |
76 | // create cookie jar | |
77 | $this->cookieJar = new CookieJar(); | |
78 | // set request options (redirect must be 0) | |
79 | $this->requestOptions = array( | |
80 | 'timeout' => 15, | |
81 | 'redirect' => 0 // we handle redirects manually so we can rewrite the new hashbang URLs that are creeping up over the web | |
82 | // TODO: test onprogress? | |
83 | ); | |
84 | if (is_array($requestOptions)) { | |
85 | $this->requestOptions = array_merge($this->requestOptions, $requestOptions); | |
86 | } | |
87 | $this->httpContext = array( | |
88 | 'http' => array( | |
89 | 'ignore_errors' => true, | |
90 | 'timeout' => $this->requestOptions['timeout'], | |
91 | 'max_redirects' => $this->requestOptions['redirect'], | |
92 | 'header' => "Accept: */*\r\n" | |
93 | ) | |
94 | ); | |
95 | } | |
96 | ||
97 | protected function debug($msg) { | |
98 | if ($this->debug) { | |
99 | $mem = round(memory_get_usage()/1024, 2); | |
100 | $memPeak = round(memory_get_peak_usage()/1024, 2); | |
101 | echo '* ',$msg; | |
102 | echo ' - mem used: ',$mem," (peak: $memPeak)\n"; | |
103 | ob_flush(); | |
104 | flush(); | |
105 | } | |
106 | } | |
107 | ||
108 | protected function getUserAgent($url, $asArray=false) { | |
109 | $host = @parse_url($url, PHP_URL_HOST); | |
110 | if (strtolower(substr($host, 0, 4)) == 'www.') { | |
111 | $host = substr($host, 4); | |
112 | } | |
113 | if ($host) { | |
114 | $try = array($host); | |
115 | $split = explode('.', $host); | |
116 | if (count($split) > 1) { | |
117 | array_shift($split); | |
118 | $try[] = '.'.implode('.', $split); | |
119 | } | |
120 | foreach ($try as $h) { | |
121 | if (isset($this->userAgentMap[$h])) { | |
122 | $ua = $this->userAgentMap[$h]; | |
123 | break; | |
124 | } | |
125 | } | |
126 | } | |
127 | if (!isset($ua)) $ua = $this->userAgentDefault; | |
128 | if ($asArray) { | |
129 | return array('User-Agent' => $ua); | |
130 | } else { | |
131 | return 'User-Agent: '.$ua; | |
132 | } | |
133 | } | |
134 | ||
135 | public function rewriteHashbangFragment($url) { | |
136 | // return $url if there's no '#!' | |
137 | if (strpos($url, '#!') === false) return $url; | |
138 | // split $url and rewrite | |
139 | // TODO: is SimplePie_IRI included? | |
140 | $iri = new SimplePie_IRI($url); | |
141 | $fragment = substr($iri->fragment, 1); // strip '!' | |
142 | $iri->fragment = null; | |
143 | if (isset($iri->query)) { | |
144 | parse_str($iri->query, $query); | |
145 | } else { | |
146 | $query = array(); | |
147 | } | |
148 | $query['_escaped_fragment_'] = (string)$fragment; | |
149 | $iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites | |
150 | return $iri->get_iri(); | |
151 | } | |
152 | ||
153 | public function removeFragment($url) { | |
154 | $pos = strpos($url, '#'); | |
155 | if ($pos === false) { | |
156 | return $url; | |
157 | } else { | |
158 | return substr($url, 0, $pos); | |
159 | } | |
160 | } | |
161 | ||
162 | public function rewriteUrls($url) { | |
163 | foreach ($this->rewriteUrls as $find => $action) { | |
164 | if (strpos($url, $find) !== false) { | |
165 | if (is_array($action)) { | |
166 | return strtr($url, $action); | |
167 | } | |
168 | } | |
169 | } | |
170 | return $url; | |
171 | } | |
172 | ||
173 | public function enableDebug($bool=true) { | |
174 | $this->debug = (bool)$bool; | |
175 | } | |
176 | ||
177 | public function minimiseMemoryUse($bool = true) { | |
178 | $this->minimiseMemoryUse = $bool; | |
179 | } | |
180 | ||
181 | public function setMaxParallelRequests($max) { | |
182 | $this->maxParallelRequests = $max; | |
183 | } | |
184 | ||
185 | public function validateUrl($url) { | |
186 | $url = filter_var($url, FILTER_SANITIZE_URL); | |
187 | $test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED); | |
188 | // deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2) | |
189 | if ($test === false) { | |
190 | $test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED); | |
191 | } | |
192 | if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) { | |
193 | return $url; | |
194 | } else { | |
195 | return false; | |
196 | } | |
197 | } | |
198 | ||
199 | public function fetchAll(array $urls) { | |
200 | $this->fetchAllOnce($urls, $isRedirect=false); | |
201 | $redirects = 0; | |
202 | while (!empty($this->redirectQueue) && ++$redirects <= $this->maxRedirects) { | |
203 | $this->debug("Following redirects #$redirects..."); | |
204 | $this->fetchAllOnce($this->redirectQueue, $isRedirect=true); | |
205 | } | |
206 | } | |
207 | ||
208 | // fetch all URLs without following redirects | |
209 | public function fetchAllOnce(array $urls, $isRedirect=false) { | |
210 | if (!$isRedirect) $urls = array_unique($urls); | |
211 | if (empty($urls)) return; | |
212 | ||
213 | ////////////////////////////////////////////////////// | |
214 | // parallel (HttpRequestPool) | |
215 | if ($this->method == self::METHOD_REQUEST_POOL) { | |
216 | $this->debug('Starting parallel fetch (HttpRequestPool)'); | |
217 | try { | |
218 | while (count($urls) > 0) { | |
219 | $this->debug('Processing set of '.min($this->maxParallelRequests, count($urls))); | |
220 | $subset = array_splice($urls, 0, $this->maxParallelRequests); | |
221 | $pool = new HttpRequestPool(); | |
222 | foreach ($subset as $orig => $url) { | |
223 | if (!$isRedirect) $orig = $url; | |
224 | unset($this->redirectQueue[$orig]); | |
225 | $this->debug("...$url"); | |
226 | if (!$isRedirect && isset($this->requests[$url])) { | |
227 | $this->debug("......in memory"); | |
228 | /* | |
229 | } elseif ($this->isCached($url)) { | |
230 | $this->debug("......is cached"); | |
231 | if (!$this->minimiseMemoryUse) { | |
232 | $this->requests[$url] = $this->getCached($url); | |
233 | } | |
234 | */ | |
235 | } else { | |
236 | $this->debug("......adding to pool"); | |
237 | $req_url = $this->rewriteUrls($url); | |
238 | $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url; | |
239 | $req_url = $this->removeFragment($req_url); | |
240 | if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) { | |
241 | $_meth = HttpRequest::METH_HEAD; | |
242 | } else { | |
243 | $_meth = HttpRequest::METH_GET; | |
244 | unset($this->requests[$orig]['wrongGuess']); | |
245 | } | |
246 | $httpRequest = new HttpRequest($req_url, $_meth, $this->requestOptions); | |
247 | // send cookies, if we have any | |
248 | if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { | |
249 | $this->debug("......sending cookies: $cookies"); | |
250 | $httpRequest->addHeaders(array('Cookie' => $cookies)); | |
251 | } | |
252 | //$httpRequest->addHeaders(array('User-Agent' => $this->userAgent)); | |
253 | $httpRequest->addHeaders($this->getUserAgent($req_url, true)); | |
254 | // add referer for picky sites | |
255 | $httpRequest->addheaders(array('Referer' => $this->referer)); | |
256 | $this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest); | |
257 | $this->requests[$orig]['original_url'] = $orig; | |
258 | $pool->attach($httpRequest); | |
259 | } | |
260 | } | |
261 | // did we get anything into the pool? | |
262 | if (count($pool) > 0) { | |
263 | $this->debug('Sending request...'); | |
264 | try { | |
265 | $pool->send(); | |
266 | } catch (HttpRequestPoolException $e) { | |
267 | // do nothing | |
268 | } | |
269 | $this->debug('Received responses'); | |
270 | foreach($subset as $orig => $url) { | |
271 | if (!$isRedirect) $orig = $url; | |
272 | $request = $this->requests[$orig]['httpRequest']; | |
273 | //$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader()); | |
274 | // getResponseHeader() doesn't return status line, so, for consistency... | |
275 | $this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size')); | |
276 | // check content type | |
277 | // TODO: use getResponseHeader('content-type') or getResponseInfo() | |
278 | if ($this->headerOnlyType($this->requests[$orig]['headers'])) { | |
279 | $this->requests[$orig]['body'] = ''; | |
280 | $_header_only_type = true; | |
281 | $this->debug('Header only type returned'); | |
282 | } else { | |
283 | $this->requests[$orig]['body'] = $request->getResponseBody(); | |
284 | $_header_only_type = false; | |
285 | } | |
286 | $this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url'); | |
287 | $this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode(); | |
288 | // is redirect? | |
289 | if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) { | |
290 | $redirectURL = $request->getResponseHeader('location'); | |
291 | if (!preg_match('!^https?://!i', $redirectURL)) { | |
292 | $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); | |
293 | } | |
294 | if ($this->validateURL($redirectURL)) { | |
295 | $this->debug('Redirect detected. Valid URL: '.$redirectURL); | |
296 | // store any cookies | |
297 | $cookies = $request->getResponseHeader('set-cookie'); | |
298 | if ($cookies && !is_array($cookies)) $cookies = array($cookies); | |
299 | if ($cookies) $this->cookieJar->storeCookies($url, $cookies); | |
300 | $this->redirectQueue[$orig] = $redirectURL; | |
301 | } else { | |
302 | $this->debug('Redirect detected. Invalid URL: '.$redirectURL); | |
303 | } | |
304 | } elseif (!$_header_only_type && $request->getMethod() === HttpRequest::METH_HEAD) { | |
305 | // the response content-type did not match our 'header only' types, | |
306 | // but we'd issues a HEAD request because we assumed it would. So | |
307 | // let's queue a proper GET request for this item... | |
308 | $this->debug('Wrong guess at content-type, queing GET request'); | |
309 | $this->requests[$orig]['wrongGuess'] = true; | |
310 | $this->redirectQueue[$orig] = $this->requests[$orig]['effective_url']; | |
311 | } | |
312 | //die($url.' -multi- '.$request->getResponseInfo('effective_url')); | |
313 | $pool->detach($request); | |
314 | unset($this->requests[$orig]['httpRequest'], $request); | |
315 | /* | |
316 | if ($this->minimiseMemoryUse) { | |
317 | if ($this->cache($url)) { | |
318 | unset($this->requests[$url]); | |
319 | } | |
320 | } | |
321 | */ | |
322 | } | |
323 | } | |
324 | } | |
325 | } catch (HttpException $e) { | |
326 | $this->debug($e); | |
327 | return false; | |
328 | } | |
329 | } | |
330 | ||
331 | ////////////////////////////////////////////////////////// | |
332 | // parallel (curl_multi_*) | |
333 | elseif ($this->method == self::METHOD_CURL_MULTI) { | |
334 | $this->debug('Starting parallel fetch (curl_multi_*)'); | |
335 | while (count($urls) > 0) { | |
336 | $this->debug('Processing set of '.min($this->maxParallelRequests, count($urls))); | |
337 | $subset = array_splice($urls, 0, $this->maxParallelRequests); | |
338 | $pool = new RollingCurl(array($this, 'handleCurlResponse')); | |
339 | $pool->window_size = count($subset); | |
340 | ||
341 | foreach ($subset as $orig => $url) { | |
342 | if (!$isRedirect) $orig = $url; | |
343 | unset($this->redirectQueue[$orig]); | |
344 | $this->debug("...$url"); | |
345 | if (!$isRedirect && isset($this->requests[$url])) { | |
346 | $this->debug("......in memory"); | |
347 | /* | |
348 | } elseif ($this->isCached($url)) { | |
349 | $this->debug("......is cached"); | |
350 | if (!$this->minimiseMemoryUse) { | |
351 | $this->requests[$url] = $this->getCached($url); | |
352 | } | |
353 | */ | |
354 | } else { | |
355 | $this->debug("......adding to pool"); | |
356 | $req_url = $this->rewriteUrls($url); | |
357 | $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url; | |
358 | $req_url = $this->removeFragment($req_url); | |
359 | if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) { | |
360 | $_meth = 'HEAD'; | |
361 | } else { | |
362 | $_meth = 'GET'; | |
363 | unset($this->requests[$orig]['wrongGuess']); | |
364 | } | |
365 | $headers = array(); | |
366 | //$headers[] = 'User-Agent: '.$this->userAgent; | |
367 | $headers[] = $this->getUserAgent($req_url); | |
368 | // add referer for picky sites | |
369 | $headers[] = 'Referer: '.$this->referer; | |
370 | // send cookies, if we have any | |
371 | if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { | |
372 | $this->debug("......sending cookies: $cookies"); | |
373 | $headers[] = 'Cookie: '.$cookies; | |
374 | } | |
375 | $httpRequest = new RollingCurlRequest($req_url, $_meth, null, $headers, array( | |
376 | CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'], | |
377 | CURLOPT_TIMEOUT => $this->requestOptions['timeout'] | |
378 | )); | |
379 | $httpRequest->set_original_url($orig); | |
380 | $this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest); | |
381 | $this->requests[$orig]['original_url'] = $orig; // TODO: is this needed anymore? | |
382 | $pool->add($httpRequest); | |
383 | } | |
384 | } | |
385 | // did we get anything into the pool? | |
386 | if (count($pool) > 0) { | |
387 | $this->debug('Sending request...'); | |
388 | $pool->execute(); // this will call handleCurlResponse() and populate $this->requests[$orig] | |
389 | $this->debug('Received responses'); | |
390 | foreach($subset as $orig => $url) { | |
391 | if (!$isRedirect) $orig = $url; | |
392 | // $this->requests[$orig]['headers'] | |
393 | // $this->requests[$orig]['body'] | |
394 | // $this->requests[$orig]['effective_url'] | |
395 | // check content type | |
396 | if ($this->headerOnlyType($this->requests[$orig]['headers'])) { | |
397 | $this->requests[$orig]['body'] = ''; | |
398 | $_header_only_type = true; | |
399 | $this->debug('Header only type returned'); | |
400 | } else { | |
401 | $_header_only_type = false; | |
402 | } | |
403 | $status_code = $this->requests[$orig]['status_code']; | |
404 | if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) { | |
405 | $redirectURL = $this->requests[$orig]['location']; | |
406 | if (!preg_match('!^https?://!i', $redirectURL)) { | |
407 | $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); | |
408 | } | |
409 | if ($this->validateURL($redirectURL)) { | |
410 | $this->debug('Redirect detected. Valid URL: '.$redirectURL); | |
411 | // store any cookies | |
412 | $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']); | |
413 | if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies); | |
414 | $this->redirectQueue[$orig] = $redirectURL; | |
415 | } else { | |
416 | $this->debug('Redirect detected. Invalid URL: '.$redirectURL); | |
417 | } | |
418 | } elseif (!$_header_only_type && $this->requests[$orig]['method'] == 'HEAD') { | |
419 | // the response content-type did not match our 'header only' types, | |
420 | // but we'd issues a HEAD request because we assumed it would. So | |
421 | // let's queue a proper GET request for this item... | |
422 | $this->debug('Wrong guess at content-type, queing GET request'); | |
423 | $this->requests[$orig]['wrongGuess'] = true; | |
424 | $this->redirectQueue[$orig] = $this->requests[$orig]['effective_url']; | |
425 | } | |
426 | // die($url.' -multi- '.$request->getResponseInfo('effective_url')); | |
427 | unset($this->requests[$orig]['httpRequest'], $this->requests[$orig]['method']); | |
428 | } | |
429 | } | |
430 | } | |
431 | } | |
432 | ||
433 | ////////////////////////////////////////////////////// | |
434 | // sequential (file_get_contents) | |
435 | else { | |
436 | $this->debug('Starting sequential fetch (file_get_contents)'); | |
437 | $this->debug('Processing set of '.count($urls)); | |
438 | foreach ($urls as $orig => $url) { | |
439 | if (!$isRedirect) $orig = $url; | |
440 | unset($this->redirectQueue[$orig]); | |
441 | $this->debug("...$url"); | |
442 | if (!$isRedirect && isset($this->requests[$url])) { | |
443 | $this->debug("......in memory"); | |
444 | /* | |
445 | } elseif ($this->isCached($url)) { | |
446 | $this->debug("......is cached"); | |
447 | if (!$this->minimiseMemoryUse) { | |
448 | $this->requests[$url] = $this->getCached($url); | |
449 | } | |
450 | */ | |
451 | } else { | |
452 | $this->debug("Sending request for $url"); | |
453 | $this->requests[$orig]['original_url'] = $orig; | |
454 | $req_url = $this->rewriteUrls($url); | |
455 | $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url; | |
456 | $req_url = $this->removeFragment($req_url); | |
457 | // send cookies, if we have any | |
458 | $httpContext = $this->httpContext; | |
459 | $httpContext['http']['header'] .= $this->getUserAgent($req_url)."\r\n"; | |
460 | // add referer for picky sites | |
461 | $httpContext['http']['header'] .= 'Referer: '.$this->referer."\r\n"; | |
462 | if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) { | |
463 | $this->debug("......sending cookies: $cookies"); | |
464 | $httpContext['http']['header'] .= 'Cookie: '.$cookies."\r\n"; | |
465 | } | |
466 | if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) { | |
467 | $this->debug('Received response'); | |
468 | // get status code | |
469 | if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\d+\.\d+\s+(\d+)!', trim($http_response_header[0]), $match)) { | |
470 | $this->debug('Error: no status code found'); | |
471 | // TODO: handle error - no status code | |
472 | } else { | |
473 | $this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false); | |
474 | // check content type | |
475 | if ($this->headerOnlyType($this->requests[$orig]['headers'])) { | |
476 | $this->requests[$orig]['body'] = ''; | |
477 | } else { | |
478 | $this->requests[$orig]['body'] = $html; | |
479 | } | |
480 | $this->requests[$orig]['effective_url'] = $req_url; | |
481 | $this->requests[$orig]['status_code'] = $status_code = (int)$match[1]; | |
482 | unset($match); | |
483 | // handle redirect | |
484 | if (preg_match('/^Location:(.*?)$/m', $this->requests[$orig]['headers'], $match)) { | |
485 | $this->requests[$orig]['location'] = trim($match[1]); | |
486 | } | |
487 | if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) { | |
488 | $redirectURL = $this->requests[$orig]['location']; | |
489 | if (!preg_match('!^https?://!i', $redirectURL)) { | |
490 | $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url); | |
491 | } | |
492 | if ($this->validateURL($redirectURL)) { | |
493 | $this->debug('Redirect detected. Valid URL: '.$redirectURL); | |
494 | // store any cookies | |
495 | $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']); | |
496 | if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies); | |
497 | $this->redirectQueue[$orig] = $redirectURL; | |
498 | } else { | |
499 | $this->debug('Redirect detected. Invalid URL: '.$redirectURL); | |
500 | } | |
501 | } | |
502 | } | |
503 | } else { | |
504 | $this->debug('Error retrieving URL'); | |
505 | //print_r($req_url); | |
506 | //print_r($http_response_header); | |
507 | //print_r($html); | |
508 | ||
509 | // TODO: handle error - failed to retrieve URL | |
510 | } | |
511 | } | |
512 | } | |
513 | } | |
514 | } | |
515 | ||
516 | public function handleCurlResponse($response, $info, $request) { | |
517 | $orig = $request->url_original; | |
518 | $this->requests[$orig]['headers'] = substr($response, 0, $info['header_size']); | |
519 | $this->requests[$orig]['body'] = substr($response, $info['header_size']); | |
520 | $this->requests[$orig]['method'] = $request->method; | |
521 | $this->requests[$orig]['effective_url'] = $info['url']; | |
522 | $this->requests[$orig]['status_code'] = (int)$info['http_code']; | |
523 | if (preg_match('/^Location:(.*?)$/m', $this->requests[$orig]['headers'], $match)) { | |
524 | $this->requests[$orig]['location'] = trim($match[1]); | |
525 | } | |
526 | } | |
527 | ||
528 | protected function headersToString(array $headers, $associative=true) { | |
529 | if (!$associative) { | |
530 | return implode("\n", $headers); | |
531 | } else { | |
532 | $str = ''; | |
533 | foreach ($headers as $key => $val) { | |
534 | if (is_array($val)) { | |
535 | foreach ($val as $v) $str .= "$key: $v\n"; | |
536 | } else { | |
537 | $str .= "$key: $val\n"; | |
538 | } | |
539 | } | |
540 | return rtrim($str); | |
541 | } | |
542 | } | |
543 | ||
544 | public function get($url, $remove=false, $gzdecode=true) { | |
545 | $url = "$url"; | |
546 | if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) { | |
547 | $this->debug("URL already fetched - in memory ($url, effective: {$this->requests[$url]['effective_url']})"); | |
548 | $response = $this->requests[$url]; | |
549 | /* | |
550 | } elseif ($this->isCached($url)) { | |
551 | $this->debug("URL already fetched - in disk cache ($url)"); | |
552 | $response = $this->getCached($url); | |
553 | $this->requests[$url] = $response; | |
554 | */ | |
555 | } else { | |
556 | $this->debug("Fetching URL ($url)"); | |
557 | $this->fetchAll(array($url)); | |
558 | if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) { | |
559 | $response = $this->requests[$url]; | |
560 | } else { | |
561 | $this->debug("Request failed"); | |
562 | $response = false; | |
563 | } | |
564 | } | |
565 | /* | |
566 | if ($this->minimiseMemoryUse && $response) { | |
567 | $this->cache($url); | |
568 | unset($this->requests[$url]); | |
569 | } | |
570 | */ | |
571 | if ($remove && $response) unset($this->requests[$url]); | |
572 | if ($gzdecode && stripos($response['headers'], 'Content-Encoding: gzip')) { | |
573 | if ($html = gzdecode($response['body'])) { | |
574 | $response['body'] = $html; | |
575 | } | |
576 | } | |
577 | return $response; | |
578 | } | |
579 | ||
580 | public function parallelSupport() { | |
581 | return class_exists('HttpRequestPool') || function_exists('curl_multi_init'); | |
582 | } | |
583 | ||
584 | private function headerOnlyType($headers) { | |
585 | if (preg_match('!^Content-Type:\s*(([a-z-]+)/([^;\r\n ]+))!im', $headers, $match)) { | |
586 | // look for full mime type (e.g. image/jpeg) or just type (e.g. image) | |
587 | $match[1] = strtolower(trim($match[1])); | |
588 | $match[2] = strtolower(trim($match[2])); | |
589 | foreach (array($match[1], $match[2]) as $mime) { | |
590 | if (in_array($mime, $this->headerOnlyTypes)) return true; | |
591 | } | |
592 | } | |
593 | return false; | |
594 | } | |
595 | ||
596 | private function possibleUnsupportedType($url) { | |
597 | $path = @parse_url($url, PHP_URL_PATH); | |
598 | if ($path && strpos($path, '.') !== false) { | |
599 | $ext = strtolower(trim(pathinfo($path, PATHINFO_EXTENSION))); | |
600 | return in_array($ext, $this->headerOnlyClues); | |
601 | } | |
602 | return false; | |
603 | } | |
604 | } | |
605 | ||
606 | // gzdecode from http://www.php.net/manual/en/function.gzdecode.php#82930 | |
607 | if (!function_exists('gzdecode')) { | |
608 | function gzdecode($data,&$filename='',&$error='',$maxlength=null) | |
609 | { | |
610 | $len = strlen($data); | |
611 | if ($len < 18 || strcmp(substr($data,0,2),"\x1f\x8b")) { | |
612 | $error = "Not in GZIP format."; | |
613 | return null; // Not GZIP format (See RFC 1952) | |
614 | } | |
615 | $method = ord(substr($data,2,1)); // Compression method | |
616 | $flags = ord(substr($data,3,1)); // Flags | |
617 | if ($flags & 31 != $flags) { | |
618 | $error = "Reserved bits not allowed."; | |
619 | return null; | |
620 | } | |
621 | // NOTE: $mtime may be negative (PHP integer limitations) | |
622 | $mtime = unpack("V", substr($data,4,4)); | |
623 | $mtime = $mtime[1]; | |
624 | $xfl = substr($data,8,1); | |
625 | $os = substr($data,8,1); | |
626 | $headerlen = 10; | |
627 | $extralen = 0; | |
628 | $extra = ""; | |
629 | if ($flags & 4) { | |
630 | // 2-byte length prefixed EXTRA data in header | |
631 | if ($len - $headerlen - 2 < 8) { | |
632 | return false; // invalid | |
633 | } | |
634 | $extralen = unpack("v",substr($data,8,2)); | |
635 | $extralen = $extralen[1]; | |
636 | if ($len - $headerlen - 2 - $extralen < 8) { | |
637 | return false; // invalid | |
638 | } | |
639 | $extra = substr($data,10,$extralen); | |
640 | $headerlen += 2 + $extralen; | |
641 | } | |
642 | $filenamelen = 0; | |
643 | $filename = ""; | |
644 | if ($flags & 8) { | |
645 | // C-style string | |
646 | if ($len - $headerlen - 1 < 8) { | |
647 | return false; // invalid | |
648 | } | |
649 | $filenamelen = strpos(substr($data,$headerlen),chr(0)); | |
650 | if ($filenamelen === false || $len - $headerlen - $filenamelen - 1 < 8) { | |
651 | return false; // invalid | |
652 | } | |
653 | $filename = substr($data,$headerlen,$filenamelen); | |
654 | $headerlen += $filenamelen + 1; | |
655 | } | |
656 | $commentlen = 0; | |
657 | $comment = ""; | |
658 | if ($flags & 16) { | |
659 | // C-style string COMMENT data in header | |
660 | if ($len - $headerlen - 1 < 8) { | |
661 | return false; // invalid | |
662 | } | |
663 | $commentlen = strpos(substr($data,$headerlen),chr(0)); | |
664 | if ($commentlen === false || $len - $headerlen - $commentlen - 1 < 8) { | |
665 | return false; // Invalid header format | |
666 | } | |
667 | $comment = substr($data,$headerlen,$commentlen); | |
668 | $headerlen += $commentlen + 1; | |
669 | } | |
670 | $headercrc = ""; | |
671 | if ($flags & 2) { | |
672 | // 2-bytes (lowest order) of CRC32 on header present | |
673 | if ($len - $headerlen - 2 < 8) { | |
674 | return false; // invalid | |
675 | } | |
676 | $calccrc = crc32(substr($data,0,$headerlen)) & 0xffff; | |
677 | $headercrc = unpack("v", substr($data,$headerlen,2)); | |
678 | $headercrc = $headercrc[1]; | |
679 | if ($headercrc != $calccrc) { | |
680 | $error = "Header checksum failed."; | |
681 | return false; // Bad header CRC | |
682 | } | |
683 | $headerlen += 2; | |
684 | } | |
685 | // GZIP FOOTER | |
686 | $datacrc = unpack("V",substr($data,-8,4)); | |
687 | $datacrc = sprintf('%u',$datacrc[1] & 0xFFFFFFFF); | |
688 | $isize = unpack("V",substr($data,-4)); | |
689 | $isize = $isize[1]; | |
690 | // decompression: | |
691 | $bodylen = $len-$headerlen-8; | |
692 | if ($bodylen < 1) { | |
693 | // IMPLEMENTATION BUG! | |
694 | return null; | |
695 | } | |
696 | $body = substr($data,$headerlen,$bodylen); | |
697 | $data = ""; | |
698 | if ($bodylen > 0) { | |
699 | switch ($method) { | |
700 | case 8: | |
701 | // Currently the only supported compression method: | |
702 | $data = gzinflate($body,$maxlength); | |
703 | break; | |
704 | default: | |
705 | $error = "Unknown compression method."; | |
706 | return false; | |
707 | } | |
708 | } // zero-byte body content is allowed | |
709 | // Verifiy CRC32 | |
710 | $crc = sprintf("%u",crc32($data)); | |
711 | $crcOK = $crc == $datacrc; | |
712 | $lenOK = $isize == strlen($data); | |
713 | if (!$lenOK || !$crcOK) { | |
714 | $error = ( $lenOK ? '' : 'Length check FAILED. ') . ( $crcOK ? '' : 'Checksum FAILED.'); | |
715 | return false; | |
716 | } | |
717 | return $data; | |
718 | } | |
719 | } | |
720 | ?> |