[change] we now use Full-Text RSS 3.1, thank you so much @fivefilters

[github/wallabag/wallabag.git] / inc / 3rdparty / libraries / humble-http-agent / HumbleHttpAgent.php

<?php\r
/**\r
 * Humble HTTP Agent\r
 * \r
 * This class is designed to take advantage of parallel HTTP requests\r
 * offered by PHP's PECL HTTP extension or the curl_multi_* functions. \r
 * For environments which do not have these options, it reverts to standard sequential \r
 * requests (using file_get_contents())\r
 * \r
 * @version 1.1\r
 * @date 2012-08-20\r
 * @see http://php.net/HttpRequestPool\r
 * @author Keyvan Minoukadeh\r
 * @copyright 2011-2012 Keyvan Minoukadeh\r
 * @license http://www.gnu.org/licenses/agpl-3.0.html AGPL v3\r
 */\r
\r
class HumbleHttpAgent\r
{\r
	const METHOD_REQUEST_POOL = 1;\r
	const METHOD_CURL_MULTI = 2;\r
	const METHOD_FILE_GET_CONTENTS = 4;\r
	//const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1';\r
	const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2';\r
	const UA_PHP = 'PHP/5.2';\r
	const REF_GOOGLE = 'http://www.google.co.uk/url?sa=t&source=web&cd=1';\r
	\r
	protected $requests = array();\r
	protected $redirectQueue = array();\r
	protected $requestOptions;\r
	protected $maxParallelRequests = 5;\r
	protected $cache = null; //TODO\r
	protected $httpContext;\r
	protected $minimiseMemoryUse = false; //TODO\r
	protected $method;\r
	protected $cookieJar;\r
	public $debug = false;\r
	public $debugVerbose = false;\r
	public $rewriteHashbangFragment = true; // see http://code.google.com/web/ajaxcrawling/docs/specification.html\r
	public $maxRedirects = 5;\r
	public $userAgentMap = array();\r
	public $rewriteUrls = array();\r
	public $userAgentDefault;\r
	public $referer;\r
	//public $userAgent = 'Mozilla/5.0';\r
	\r
	// Prevent certain file/mime types\r
	// HTTP responses which match these content types will\r
	// be returned without body.\r
	public $headerOnlyTypes = array();\r
	// URLs ending with one of these extensions will\r
	// prompt Humble HTTP Agent to send a HEAD request first\r
	// to see if returned content type matches $headerOnlyTypes.\r
	public $headerOnlyClues = array('pdf','mp3','zip','exe','gif','gzip','gz','jpeg','jpg','mpg','mpeg','png','ppt','mov');\r
	// AJAX triggers to search for.\r
	// for AJAX sites, e.g. Blogger with its dynamic views templates.\r
	public $ajaxTriggers = array("<meta name='fragment' content='!'",'<meta name="fragment" content="!"',"<meta content='!' name='fragment'",'<meta content="!" name="fragment"');\r
	\r
	//TODO: set max file size\r
	//TODO: normalise headers\r
	\r
	function __construct($requestOptions=null, $method=null) {\r
		$this->userAgentDefault = self::UA_BROWSER;\r
		$this->referer = self::REF_GOOGLE;\r
		// set the request method\r
		if (in_array($method, array(1,2,4))) {\r
			$this->method = $method;\r
		} else {\r
			if (class_exists('HttpRequestPool')) {\r
				$this->method = self::METHOD_REQUEST_POOL;\r
			} elseif (function_exists('curl_multi_init')) {\r
				$this->method = self::METHOD_CURL_MULTI;\r
			} else {\r
				$this->method = self::METHOD_FILE_GET_CONTENTS;\r
			}\r
		}\r
		if ($this->method == self::METHOD_CURL_MULTI) {\r
			require_once(dirname(__FILE__).'/RollingCurl.php');\r
		}\r
		// create cookie jar\r
		$this->cookieJar = new CookieJar();\r
		// set request options (redirect must be 0)\r
		$this->requestOptions = array(\r
			'timeout' => 15,\r
			'redirect' => 0 // we handle redirects manually so we can rewrite the new hashbang URLs that are creeping up over the web\r
			// TODO: test onprogress?\r
		);\r
		if (is_array($requestOptions)) {\r
			$this->requestOptions = array_merge($this->requestOptions, $requestOptions);\r
		}\r
		$this->httpContext = array(\r
			'http' => array(\r
				'ignore_errors' => true,\r
				'timeout' => $this->requestOptions['timeout'],\r
				'max_redirects' => $this->requestOptions['redirect'],\r
				'header' => "Accept: */*\r\n"\r
				)\r
			);\r
	}\r
	\r
	protected function debug($msg) {\r
		if ($this->debug) {\r
			$mem = round(memory_get_usage()/1024, 2);\r
			$memPeak = round(memory_get_peak_usage()/1024, 2);\r
			echo '* ',$msg;\r
			if ($this->debugVerbose) echo ' - mem used: ',$mem," (peak: $memPeak)";\r
			echo "\n";\r
			ob_flush();\r
			flush();\r
		}\r
	}\r
	\r
	protected function getUserAgent($url, $asArray=false) {\r
		$host = @parse_url($url, PHP_URL_HOST);\r
		if (strtolower(substr($host, 0, 4)) == 'www.') {\r
			$host = substr($host, 4);\r
		}\r
		if ($host) {\r
			$try = array($host);\r
			$split = explode('.', $host);\r
			if (count($split) > 1) {\r
				array_shift($split);\r
				$try[] = '.'.implode('.', $split);\r
			}\r
			foreach ($try as $h) {\r
				if (isset($this->userAgentMap[$h])) {\r
					$ua = $this->userAgentMap[$h];\r
					break;\r
				}\r
			}\r
		}\r
		if (!isset($ua)) $ua = $this->userAgentDefault;\r
		if ($asArray) {\r
			return array('User-Agent' => $ua);\r
		} else {\r
			return 'User-Agent: '.$ua;\r
		}\r
	}\r
	\r
	public function rewriteHashbangFragment($url) {\r
		// return $url if there's no '#!'\r
		if (strpos($url, '#!') === false) return $url;\r
		// split $url and rewrite\r
		// TODO: is SimplePie_IRI included?\r
		$iri = new SimplePie_IRI($url);\r
		$fragment = substr($iri->fragment, 1); // strip '!'\r
		$iri->fragment = null;\r
		if (isset($iri->query)) {\r
			parse_str($iri->query, $query);\r
		} else {\r
			$query = array();\r
		}\r
		$query['_escaped_fragment_'] = (string)$fragment;\r
		$iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites\r
		return $iri->get_iri();\r
	}\r
	\r
	public function getUglyURL($url, $html) {\r
		if ($html == '') return false;\r
		$found = false;\r
		foreach ($this->ajaxTriggers as $string) {\r
			if (stripos($html, $string)) {\r
				$found = true;\r
				break;\r
			}\r
		}\r
		if (!$found) return false;\r
		$iri = new SimplePie_IRI($url);\r
		if (isset($iri->query)) {\r
			parse_str($iri->query, $query);\r
		} else {\r
			$query = array();\r
		}\r
		$query['_escaped_fragment_'] = '';\r
		$iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites\r
		return $iri->get_iri();\r
	}\r
	\r
	public function removeFragment($url) {\r
		$pos = strpos($url, '#');\r
		if ($pos === false) {\r
			return $url;\r
		} else {\r
			return substr($url, 0, $pos);\r
		}\r
	}\r
	\r
	public function rewriteUrls($url) {\r
		foreach ($this->rewriteUrls as $find => $action) {\r
			if (strpos($url, $find) !== false) {\r
				if (is_array($action)) {\r
					return strtr($url, $action);\r
				}\r
			}\r
		}\r
		return $url;\r
	}\r
	\r
	public function enableDebug($bool=true) {\r
		$this->debug = (bool)$bool;\r
	}\r
	\r
	public function minimiseMemoryUse($bool = true) {\r
		$this->minimiseMemoryUse = $bool;\r
	}\r
	\r
	public function setMaxParallelRequests($max) {\r
		$this->maxParallelRequests = $max;\r
	}\r
	\r
	public function validateUrl($url) {\r
		$url = filter_var($url, FILTER_SANITIZE_URL);\r
		$test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);\r
		// deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2)\r
		if ($test === false) {\r
			$test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);\r
		}\r
		if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) {\r
			return $url;\r
		} else {\r
			return false;\r
		}\r
	}\r
	\r
	public function fetchAll(array $urls) {\r
		$this->fetchAllOnce($urls, $isRedirect=false);\r
		$redirects = 0;\r
		while (!empty($this->redirectQueue) && ++$redirects <= $this->maxRedirects) {\r
			$this->debug("Following redirects #$redirects...");\r
			$this->fetchAllOnce($this->redirectQueue, $isRedirect=true);\r
		}\r
	}\r
	\r
	// fetch all URLs without following redirects\r
	public function fetchAllOnce(array $urls, $isRedirect=false) {\r
		if (!$isRedirect) $urls = array_unique($urls);\r
		if (empty($urls)) return;\r
		\r
		//////////////////////////////////////////////////////\r
		// parallel (HttpRequestPool)\r
		if ($this->method == self::METHOD_REQUEST_POOL) {\r
			$this->debug('Starting parallel fetch (HttpRequestPool)');\r
			try {\r
				while (count($urls) > 0) {\r
					$this->debug('Processing set of '.min($this->maxParallelRequests, count($urls)));\r
					$subset = array_splice($urls, 0, $this->maxParallelRequests);\r
					$pool = new HttpRequestPool();\r
					foreach ($subset as $orig => $url) {\r
						if (!$isRedirect) $orig = $url;\r
						unset($this->redirectQueue[$orig]);\r
						$this->debug("...$url");\r
						if (!$isRedirect && isset($this->requests[$url])) {\r
							$this->debug("......in memory");\r
						/*\r
						} elseif ($this->isCached($url)) {\r
							$this->debug("......is cached");\r
							if (!$this->minimiseMemoryUse) {\r
								$this->requests[$url] = $this->getCached($url);\r
							}\r
						*/\r
						} else {\r
							$this->debug("......adding to pool");\r
							$req_url = $this->rewriteUrls($url);\r
							$req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;\r
							$req_url = $this->removeFragment($req_url);\r
							if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) {\r
								$_meth = HttpRequest::METH_HEAD;\r
							} else {\r
								$_meth = HttpRequest::METH_GET;\r
								unset($this->requests[$orig]['wrongGuess']);\r
							}\r
							$httpRequest = new HttpRequest($req_url, $_meth, $this->requestOptions);\r
							// send cookies, if we have any\r
							if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {\r
								$this->debug("......sending cookies: $cookies");\r
								$httpRequest->addHeaders(array('Cookie' => $cookies));\r
							}\r
							//$httpRequest->addHeaders(array('User-Agent' => $this->userAgent));\r
							$httpRequest->addHeaders($this->getUserAgent($req_url, true));\r
							// add referer for picky sites\r
							$httpRequest->addheaders(array('Referer' => $this->referer));\r
							$this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest);\r
							$this->requests[$orig]['original_url'] = $orig;\r
							$pool->attach($httpRequest);\r
						}\r
					}\r
					// did we get anything into the pool?\r
					if (count($pool) > 0) {\r
						$this->debug('Sending request...');\r
						try {\r
							$pool->send();\r
						} catch (HttpRequestPoolException $e) {\r
							// do nothing\r
						}\r
						$this->debug('Received responses');\r
						foreach($subset as $orig => $url) {\r
							if (!$isRedirect) $orig = $url;\r
							$request = $this->requests[$orig]['httpRequest'];\r
							//$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader());\r
							// getResponseHeader() doesn't return status line, so, for consistency...\r
							$this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size'));\r
							// check content type\r
							// TODO: use getResponseHeader('content-type') or getResponseInfo()\r
							if ($this->headerOnlyType($this->requests[$orig]['headers'])) {\r
								$this->requests[$orig]['body'] = '';\r
								$_header_only_type = true;\r
								$this->debug('Header only type returned');\r
							} else {\r
								$this->requests[$orig]['body'] = $request->getResponseBody();\r
								$_header_only_type = false;\r
							}\r
							$this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url');\r
							$this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode();\r
							// is redirect?\r
							if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) {\r
								$redirectURL = $request->getResponseHeader('location');\r
								if (!preg_match('!^https?://!i', $redirectURL)) {\r
									$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);\r
								}\r
								if ($this->validateURL($redirectURL)) {\r
									$this->debug('Redirect detected. Valid URL: '.$redirectURL);\r
									// store any cookies\r
									$cookies = $request->getResponseHeader('set-cookie');\r
									if ($cookies && !is_array($cookies)) $cookies = array($cookies);\r
									if ($cookies) $this->cookieJar->storeCookies($url, $cookies);\r
									$this->redirectQueue[$orig] = $redirectURL;\r
								} else {\r
									$this->debug('Redirect detected. Invalid URL: '.$redirectURL);\r
								}\r
							} elseif (!$_header_only_type && $request->getMethod() === HttpRequest::METH_HEAD) {\r
								// the response content-type did not match our 'header only' types, \r
								// but we'd issues a HEAD request because we assumed it would. So\r
								// let's queue a proper GET request for this item...\r
								$this->debug('Wrong guess at content-type, queing GET request');\r
								$this->requests[$orig]['wrongGuess'] = true;\r
								$this->redirectQueue[$orig] = $this->requests[$orig]['effective_url'];\r
							} elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {\r
								// check for <meta name='fragment' content='!'/>\r
								// for AJAX sites, e.g. Blogger with its dynamic views templates.\r
								// Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification\r
								if (isset($this->requests[$orig]['body'])) {\r
									$redirectURL = $this->getUglyURL($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));\r
									if ($redirectURL) {\r
										$this->debug('AJAX trigger (meta name="fragment" content="!") found. Queueing '.$redirectURL);\r
										$this->redirectQueue[$orig] = $redirectURL;\r
									}\r
								}\r
							}\r
							//die($url.' -multi- '.$request->getResponseInfo('effective_url'));\r
							$pool->detach($request);\r
							unset($this->requests[$orig]['httpRequest'], $request);\r
							/*\r
							if ($this->minimiseMemoryUse) {\r
								if ($this->cache($url)) {\r
									unset($this->requests[$url]);\r
								}\r
							}\r
							*/\r
						}\r
					}\r
				}\r
			} catch (HttpException $e) {\r
				$this->debug($e);\r
				return false;\r
			}\r
		}\r
		\r
		//////////////////////////////////////////////////////////\r
		// parallel (curl_multi_*)\r
		elseif ($this->method == self::METHOD_CURL_MULTI) {\r
			$this->debug('Starting parallel fetch (curl_multi_*)');\r
			while (count($urls) > 0) {\r
				$this->debug('Processing set of '.min($this->maxParallelRequests, count($urls)));\r
				$subset = array_splice($urls, 0, $this->maxParallelRequests);\r
				$pool = new RollingCurl(array($this, 'handleCurlResponse'));\r
				$pool->window_size = count($subset);		\r
				\r
				foreach ($subset as $orig => $url) {\r
					if (!$isRedirect) $orig = $url;\r
					unset($this->redirectQueue[$orig]);\r
					$this->debug("...$url");\r
					if (!$isRedirect && isset($this->requests[$url])) {\r
						$this->debug("......in memory");\r
					/*\r
					} elseif ($this->isCached($url)) {\r
						$this->debug("......is cached");\r
						if (!$this->minimiseMemoryUse) {\r
							$this->requests[$url] = $this->getCached($url);\r
						}\r
					*/\r
					} else {\r
						$this->debug("......adding to pool");\r
						$req_url = $this->rewriteUrls($url);\r
						$req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;\r
						$req_url = $this->removeFragment($req_url);\r
						if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) {\r
							$_meth = 'HEAD';\r
						} else {\r
							$_meth = 'GET';\r
							unset($this->requests[$orig]['wrongGuess']);\r
						}						\r
						$headers = array();\r
						//$headers[] = 'User-Agent: '.$this->userAgent;\r
						$headers[] = $this->getUserAgent($req_url);\r
						// add referer for picky sites\r
						$headers[] = 'Referer: '.$this->referer;\r
						// send cookies, if we have any\r
						if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {\r
							$this->debug("......sending cookies: $cookies");\r
							$headers[] = 'Cookie: '.$cookies;\r
						}\r
						$httpRequest = new RollingCurlRequest($req_url, $_meth, null, $headers, array(\r
							CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'],\r
							CURLOPT_TIMEOUT => $this->requestOptions['timeout']\r
							));\r
						$httpRequest->set_original_url($orig);\r
						$this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest);\r
						$this->requests[$orig]['original_url'] = $orig; // TODO: is this needed anymore?\r
						$pool->add($httpRequest);\r
					}\r
				}\r
				// did we get anything into the pool?\r
				if (count($pool) > 0) {\r
					$this->debug('Sending request...');\r
					$pool->execute(); // this will call handleCurlResponse() and populate $this->requests[$orig]\r
					$this->debug('Received responses');\r
					foreach($subset as $orig => $url) {\r
						if (!$isRedirect) $orig = $url;\r
						// $this->requests[$orig]['headers']\r
						// $this->requests[$orig]['body']\r
						// $this->requests[$orig]['effective_url']\r
						// check content type\r
						if ($this->headerOnlyType($this->requests[$orig]['headers'])) {\r
							$this->requests[$orig]['body'] = '';\r
							$_header_only_type = true;\r
							$this->debug('Header only type returned');\r
						} else {\r
							$_header_only_type = false;\r
						}\r
						$status_code = $this->requests[$orig]['status_code'];\r
						if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {\r
							$redirectURL = $this->requests[$orig]['location'];\r
							if (!preg_match('!^https?://!i', $redirectURL)) {\r
								$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);\r
							}\r
							if ($this->validateURL($redirectURL)) {\r
								$this->debug('Redirect detected. Valid URL: '.$redirectURL);\r
								// store any cookies\r
								$cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);\r
								if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies);							\r
								$this->redirectQueue[$orig] = $redirectURL;\r
							} else {\r
								$this->debug('Redirect detected. Invalid URL: '.$redirectURL);\r
							}\r
						} elseif (!$_header_only_type && $this->requests[$orig]['method'] == 'HEAD') {\r
							// the response content-type did not match our 'header only' types, \r
							// but we'd issues a HEAD request because we assumed it would. So\r
							// let's queue a proper GET request for this item...\r
							$this->debug('Wrong guess at content-type, queing GET request');\r
							$this->requests[$orig]['wrongGuess'] = true;\r
							$this->redirectQueue[$orig] = $this->requests[$orig]['effective_url'];\r
						} elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {\r
							// check for <meta name='fragment' content='!'/>\r
							// for AJAX sites, e.g. Blogger with its dynamic views templates.\r
							// Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification\r
							if (isset($this->requests[$orig]['body'])) {\r
								$redirectURL = $this->getUglyURL($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));\r
								if ($redirectURL) {\r
									$this->debug('AJAX trigger (meta name="fragment" content="!") found. Queueing '.$redirectURL);\r
									$this->redirectQueue[$orig] = $redirectURL;\r
								}\r
							}\r
						}\r
						// die($url.' -multi- '.$request->getResponseInfo('effective_url'));\r
						unset($this->requests[$orig]['httpRequest'], $this->requests[$orig]['method']);\r
					}\r
				}\r
			}\r
		}\r
\r
		//////////////////////////////////////////////////////\r
		// sequential (file_get_contents)\r
		else {\r
			$this->debug('Starting sequential fetch (file_get_contents)');\r
			$this->debug('Processing set of '.count($urls));\r
			foreach ($urls as $orig => $url) {\r
				if (!$isRedirect) $orig = $url;\r
				unset($this->redirectQueue[$orig]);\r
				$this->debug("...$url");\r
				if (!$isRedirect && isset($this->requests[$url])) {\r
					$this->debug("......in memory");\r
				/*\r
				} elseif ($this->isCached($url)) {\r
					$this->debug("......is cached");\r
					if (!$this->minimiseMemoryUse) {\r
						$this->requests[$url] = $this->getCached($url);\r
					}\r
				*/\r
				} else {\r
					$this->debug("Sending request for $url");\r
					$this->requests[$orig]['original_url'] = $orig;\r
					$req_url = $this->rewriteUrls($url);\r
					$req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;\r
					$req_url = $this->removeFragment($req_url);\r
					// send cookies, if we have any\r
					$httpContext = $this->httpContext;\r
					$httpContext['http']['header'] .= $this->getUserAgent($req_url)."\r\n";\r
					// add referer for picky sites\r
					$httpContext['http']['header'] .= 'Referer: '.$this->referer."\r\n";\r
					if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {\r
						$this->debug("......sending cookies: $cookies");\r
						$httpContext['http']['header'] .= 'Cookie: '.$cookies."\r\n";\r
					}\r
					if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) {\r
						$this->debug('Received response');\r
						// get status code\r
						if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\d+\.\d+\s+(\d+)!', trim($http_response_header[0]), $match)) {\r
							$this->debug('Error: no status code found');\r
							// TODO: handle error - no status code\r
						} else {\r
							$this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false);\r
							// check content type\r
							if ($this->headerOnlyType($this->requests[$orig]['headers'])) {\r
								$this->requests[$orig]['body'] = '';\r
							} else {\r
								$this->requests[$orig]['body'] = $html;\r
							}\r
							$this->requests[$orig]['effective_url'] = $req_url;\r
							$this->requests[$orig]['status_code'] = $status_code = (int)$match[1];\r
							unset($match);\r
							// handle redirect\r
							if (preg_match('/^Location:(.*?)$/mi', $this->requests[$orig]['headers'], $match)) {\r
								$this->requests[$orig]['location'] =  trim($match[1]);\r
							}\r
							if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {\r
								$redirectURL = $this->requests[$orig]['location'];\r
								if (!preg_match('!^https?://!i', $redirectURL)) {\r
									$redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);\r
								}\r
								if ($this->validateURL($redirectURL)) {\r
									$this->debug('Redirect detected. Valid URL: '.$redirectURL);\r
									// store any cookies\r
									$cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);\r
									if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies);\r
									$this->redirectQueue[$orig] = $redirectURL;\r
								} else {\r
									$this->debug('Redirect detected. Invalid URL: '.$redirectURL);\r
								}\r
							} elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {\r
								// check for <meta name='fragment' content='!'/>\r
								// for AJAX sites, e.g. Blogger with its dynamic views templates.\r
								// Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification\r
								if (isset($this->requests[$orig]['body'])) {\r
									$redirectURL = $this->getUglyURL($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));\r
									if ($redirectURL) {\r
										$this->debug('AJAX trigger (meta name="fragment" content="!") found. Queueing '.$redirectURL);\r
										$this->redirectQueue[$orig] = $redirectURL;\r
									}\r
								}\r
							}\r
						}\r
					} else {\r
						$this->debug('Error retrieving URL');\r
						//print_r($req_url);\r
						//print_r($http_response_header);\r
						//print_r($html);\r
						\r
						// TODO: handle error - failed to retrieve URL\r
					}\r
				}\r
			}\r
		}\r
	}\r
	\r
	public function handleCurlResponse($response, $info, $request) {\r
		$orig = $request->url_original;\r
		$this->requests[$orig]['headers'] = substr($response, 0, $info['header_size']);\r
		$this->requests[$orig]['body'] = substr($response, $info['header_size']);\r
		$this->requests[$orig]['method'] = $request->method;\r
		$this->requests[$orig]['effective_url'] = $info['url'];\r
		$this->requests[$orig]['status_code'] = (int)$info['http_code'];\r
		if (preg_match('/^Location:(.*?)$/mi', $this->requests[$orig]['headers'], $match)) {\r
			$this->requests[$orig]['location'] =  trim($match[1]);\r
		}\r
	}\r
	\r
	protected function headersToString(array $headers, $associative=true) {\r
		if (!$associative) {\r
			return implode("\n", $headers);\r
		} else {\r
			$str = '';\r
			foreach ($headers as $key => $val) {\r
				if (is_array($val)) {\r
					foreach ($val as $v) $str .= "$key: $v\n";\r
				} else {\r
					$str .= "$key: $val\n";\r
				}\r
			}\r
			return rtrim($str);\r
		}\r
	}\r
	\r
	public function get($url, $remove=false, $gzdecode=true) {\r
		$url = "$url";\r
		if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) {\r
			$this->debug("URL already fetched - in memory ($url, effective: {$this->requests[$url]['effective_url']})");\r
			$response = $this->requests[$url];\r
		/*\r
		} elseif ($this->isCached($url)) {\r
			$this->debug("URL already fetched - in disk cache ($url)");\r
			$response = $this->getCached($url);\r
			$this->requests[$url] = $response;\r
		*/\r
		} else {\r
			$this->debug("Fetching URL ($url)");\r
			$this->fetchAll(array($url));\r
			if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) {\r
				$response = $this->requests[$url];\r
			} else {\r
				$this->debug("Request failed");\r
				$response = false;\r
			}\r
		}\r
		/*\r
		if ($this->minimiseMemoryUse && $response) {\r
			$this->cache($url);\r
			unset($this->requests[$url]);\r
		}\r
		*/\r
		if ($remove && $response) unset($this->requests[$url]);\r
		if ($gzdecode && stripos($response['headers'], 'Content-Encoding: gzip')) {\r
			if ($html = gzdecode($response['body'])) {\r
				$response['body'] = $html;\r
			}\r
		}\r
		return $response;\r
	}\r
	\r
	public function parallelSupport() {\r
		return class_exists('HttpRequestPool') || function_exists('curl_multi_init');\r
	}\r
	\r
	private function headerOnlyType($headers) {\r
		if (preg_match('!^Content-Type:\s*(([a-z-]+)/([^;\r\n ]+))!im', $headers, $match)) {\r
			// look for full mime type (e.g. image/jpeg) or just type (e.g. image)\r
			$match[1] = strtolower(trim($match[1]));\r
			$match[2] = strtolower(trim($match[2]));\r
			foreach (array($match[1], $match[2]) as $mime) {\r
				if (in_array($mime, $this->headerOnlyTypes)) return true;\r
			}\r
		}\r
		return false;\r
	}\r
	\r
	private function possibleUnsupportedType($url) {\r
		$path = @parse_url($url, PHP_URL_PATH);\r
		if ($path && strpos($path, '.') !== false) {\r
			$ext = strtolower(trim(pathinfo($path, PATHINFO_EXTENSION)));\r
			return in_array($ext, $this->headerOnlyClues);\r
		}\r
		return false;\r
	}\r
}\r
\r
// gzdecode from http://www.php.net/manual/en/function.gzdecode.php#82930\r
if (!function_exists('gzdecode')) {\r
	function gzdecode($data,&$filename='',&$error='',$maxlength=null) \r
	{\r
		$len = strlen($data);\r
		if ($len < 18 || strcmp(substr($data,0,2),"\x1f\x8b")) {\r
			$error = "Not in GZIP format.";\r
			return null;  // Not GZIP format (See RFC 1952)\r
		}\r
		$method = ord(substr($data,2,1));  // Compression method\r
		$flags  = ord(substr($data,3,1));  // Flags\r
		if ($flags & 31 != $flags) {\r
			$error = "Reserved bits not allowed.";\r
			return null;\r
		}\r
		// NOTE: $mtime may be negative (PHP integer limitations)\r
		$mtime = unpack("V", substr($data,4,4));\r
		$mtime = $mtime[1];\r
		$xfl   = substr($data,8,1);\r
		$os    = substr($data,8,1);\r
		$headerlen = 10;\r
		$extralen  = 0;\r
		$extra     = "";\r
		if ($flags & 4) {\r
			// 2-byte length prefixed EXTRA data in header\r
			if ($len - $headerlen - 2 < 8) {\r
				return false;  // invalid\r
			}\r
			$extralen = unpack("v",substr($data,8,2));\r
			$extralen = $extralen[1];\r
			if ($len - $headerlen - 2 - $extralen < 8) {\r
				return false;  // invalid\r
			}\r
			$extra = substr($data,10,$extralen);\r
			$headerlen += 2 + $extralen;\r
		}\r
		$filenamelen = 0;\r
		$filename = "";\r
		if ($flags & 8) {\r
			// C-style string\r
			if ($len - $headerlen - 1 < 8) {\r
				return false; // invalid\r
			}\r
			$filenamelen = strpos(substr($data,$headerlen),chr(0));\r
			if ($filenamelen === false || $len - $headerlen - $filenamelen - 1 < 8) {\r
				return false; // invalid\r
			}\r
			$filename = substr($data,$headerlen,$filenamelen);\r
			$headerlen += $filenamelen + 1;\r
		}\r
		$commentlen = 0;\r
		$comment = "";\r
		if ($flags & 16) {\r
			// C-style string COMMENT data in header\r
			if ($len - $headerlen - 1 < 8) {\r
				return false;    // invalid\r
			}\r
			$commentlen = strpos(substr($data,$headerlen),chr(0));\r
			if ($commentlen === false || $len - $headerlen - $commentlen - 1 < 8) {\r
				return false;    // Invalid header format\r
			}\r
			$comment = substr($data,$headerlen,$commentlen);\r
			$headerlen += $commentlen + 1;\r
		}\r
		$headercrc = "";\r
		if ($flags & 2) {\r
			// 2-bytes (lowest order) of CRC32 on header present\r
			if ($len - $headerlen - 2 < 8) {\r
				return false;    // invalid\r
			}\r
			$calccrc = crc32(substr($data,0,$headerlen)) & 0xffff;\r
			$headercrc = unpack("v", substr($data,$headerlen,2));\r
			$headercrc = $headercrc[1];\r
			if ($headercrc != $calccrc) {\r
				$error = "Header checksum failed.";\r
				return false;    // Bad header CRC\r
			}\r
			$headerlen += 2;\r
		}\r
		// GZIP FOOTER\r
		$datacrc = unpack("V",substr($data,-8,4));\r
		$datacrc = sprintf('%u',$datacrc[1] & 0xFFFFFFFF);\r
		$isize = unpack("V",substr($data,-4));\r
		$isize = $isize[1];\r
		// decompression:\r
		$bodylen = $len-$headerlen-8;\r
		if ($bodylen < 1) {\r
			// IMPLEMENTATION BUG!\r
			return null;\r
		}\r
		$body = substr($data,$headerlen,$bodylen);\r
		$data = "";\r
		if ($bodylen > 0) {\r
			switch ($method) {\r
			case 8:\r
				// Currently the only supported compression method:\r
				$data = gzinflate($body,$maxlength);\r
				break;\r
			default:\r
				$error = "Unknown compression method.";\r
				return false;\r
			}\r
		}  // zero-byte body content is allowed\r
		// Verifiy CRC32\r
		$crc   = sprintf("%u",crc32($data));\r
		$crcOK = $crc == $datacrc;\r
		$lenOK = $isize == strlen($data);\r
		if (!$lenOK || !$crcOK) {\r
			$error = ( $lenOK ? '' : 'Length check FAILED. ') . ( $crcOK ? '' : 'Checksum FAILED.');\r
			return false;\r
		}\r
		return $data;\r
	}\r
}\r
?>