Merge pull request #707 from mariroz/dev

update to 3.2 version of full-text-rss, issue #694
author: Nicolas Lœuillet <nicolas@loeuillet.org> 2014-05-29 12:50:28 +0200
committer: Nicolas Lœuillet <nicolas@loeuillet.org> 2014-05-29 12:50:28 +0200
commit: 87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b (patch)
tree: 558818975ac41403e7d55ad07c5b0ac29806e907 /inc/3rdparty/libraries/humble-http-agent
parent: ab157bbb75ba226917145c9bf906cbf764a85cd0 (diff)
parent: 0b9bb8cb7868f24137c5d8b85c39cc88ea877411 (diff)
download: wallabag-87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b.tar.gz
wallabag-87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b.tar.zst
wallabag-87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b.zip
3 files changed, 1291 insertions, 1262 deletions
diff --git a/inc/3rdparty/libraries/humble-http-agent/CookieJar.php b/inc/3rdparty/libraries/humble-http-agent/CookieJar.php
index 83e94f14..e4d5f495 100644
--- a/inc/3rdparty/libraries/humble-http-agent/CookieJar.php
+++ b/inc/3rdparty/libraries/humble-http-agent/CookieJar.php
@@ -1,404 +1,403 @@
-<?php
+<?php
-/**
+/**
- * Cookie Jar
+ * Cookie Jar
- * 
+ * 
- * PHP class for handling cookies, as defined by the Netscape spec: 
+ * PHP class for handling cookies, as defined by the Netscape spec: 
- * <http://curl.haxx.se/rfc/cookie_spec.html>
+ * <http://curl.haxx.se/rfc/cookie_spec.html>
- *
+ *
- * This class should be used to handle cookies (storing cookies from HTTP response messages, and
+ * This class should be used to handle cookies (storing cookies from HTTP response messages, and
- * sending out cookies in HTTP request messages). This has been adapted for FiveFilters.org 
+ * sending out cookies in HTTP request messages). This has been adapted for FiveFilters.org 
- * from the original version used in HTTP Navigator. See http://www.keyvan.net/code/http-navigator/
+ * from the original version used in HTTP Navigator. See http://www.keyvan.net/code/http-navigator/
- * 
+ * 
- * This class is mainly based on Cookies.pm <http://search.cpan.org/author/GAAS/libwww-perl-5.65/
+ * This class is mainly based on Cookies.pm <http://search.cpan.org/author/GAAS/libwww-perl-5.65/
- * lib/HTTP/Cookies.pm> from the libwww-perl collection <http://www.linpro.no/lwp/>.
+ * lib/HTTP/Cookies.pm> from the libwww-perl collection <http://www.linpro.no/lwp/>.
- * Unlike Cookies.pm, this class only supports the Netscape cookie spec, not RFC 2965.
+ * Unlike Cookies.pm, this class only supports the Netscape cookie spec, not RFC 2965.
- * 
+ * 
- * @version 0.5
+ * @version 0.5
- * @date 2011-03-15
+ * @date 2011-03-15
- * @see http://php.net/HttpRequestPool
+ * @see http://php.net/HttpRequestPool
- * @author Keyvan Minoukadeh
+ * @author Keyvan Minoukadeh
- * @copyright 2011 Keyvan Minoukadeh
+ * @copyright 2011 Keyvan Minoukadeh
- * @license http://www.gnu.org/licenses/agpl-3.0.html AGPL v3
+ * @license http://www.gnu.org/licenses/agpl-3.0.html AGPL v3
- */
+ */
-class CookieJar
+class CookieJar
-{
+{
-    /**
+    /**
-    * Cookies - array containing all cookies.
+    * Cookies - array containing all cookies.
-    *
+    *
-    * <pre>
+    * <pre>
-    * Cookies are stored like this:
+    * Cookies are stored like this:
-    *   [domain][path][name] = array
+    *   [domain][path][name] = array
-    * where array is:
+    * where array is:
-    *   0 => value, 1 => secure, 2 => expires
+    *   0 => value, 1 => secure, 2 => expires
-    * </pre>
+    * </pre>
-    * @var array
+    * @var array
-    * @access private
+    * @access private
-    */
+    */
-    public $cookies = array();
+    public $cookies = array();
-        public $debug = false;
+        public $debug = false;
-    /**
+    /**
-    * Constructor
+    * Constructor
-    */
+    */
-    function __construct() {
+    function __construct() {
-    }
+    }
-        protected function debug($msg, $file=null, $line=null) {
+        protected function debug($msg, $file=null, $line=null) {
-                if ($this->debug) {
+                if ($this->debug) {
-                        $mem = round(memory_get_usage()/1024, 2);
+                        $mem = round(memory_get_usage()/1024, 2);
-                        $memPeak = round(memory_get_peak_usage()/1024, 2);
+                        $memPeak = round(memory_get_peak_usage()/1024, 2);
-                        echo '* ',$msg;
+                        echo '* ',$msg;
-                        if (isset($file, $line)) echo " ($file line $line)";
+                        if (isset($file, $line)) echo " ($file line $line)";
-                        echo ' - mem used: ',$mem," (peak: $memPeak)\n";        
+                        echo ' - mem used: ',$mem," (peak: $memPeak)\n";        
-                        ob_flush();
+                        ob_flush();
-                        flush();
+                        flush();
-                }
+                }
-        }       
+        }       
-        
+        
-    /**
+    /**
-    * Get matching cookies
+    * Get matching cookies
-    *
+    *
-    * Only use this method if you cannot use add_cookie_header(), for example, if you want to use
+    * Only use this method if you cannot use add_cookie_header(), for example, if you want to use
-    * this cookie jar class without using the request class.
+    * this cookie jar class without using the request class.
-    *
+    *
-    * @param array $param associative array containing 'domain', 'path', 'secure' keys
+    * @param array $param associative array containing 'domain', 'path', 'secure' keys
-    * @return string
+    * @return string
-    * @see add_cookie_header()
+    * @see add_cookie_header()
-    */
+    */
-    public function getMatchingCookies($url)
+    public function getMatchingCookies($url)
-    {
+    {
-                if (($parts = @parse_url($url)) && isset($parts['scheme'], $parts['host'], $parts['path'])) {
+                if (($parts = @parse_url($url)) && isset($parts['scheme'], $parts['host'], $parts['path'])) {
-                        $param['domain'] = $parts['host'];
+                        $param['domain'] = $parts['host'];
-                        $param['path'] = $parts['path'];
+                        $param['path'] = $parts['path'];
-                        $param['secure'] = (strtolower($parts['scheme']) == 'https');
+                        $param['secure'] = (strtolower($parts['scheme']) == 'https');
-                        unset($parts);
+                        unset($parts);
-                } else {
+                } else {
-                        return false;
+                        return false;
-                }
+                }
-        // RFC 2965 notes:
+        // RFC 2965 notes:
-        //  If multiple cookies satisfy the criteria above, they are ordered in
+        //  If multiple cookies satisfy the criteria above, they are ordered in
-        //  the Cookie header such that those with more specific Path attributes
+        //  the Cookie header such that those with more specific Path attributes
-        //  precede those with less specific.  Ordering with respect to other
+        //  precede those with less specific.  Ordering with respect to other
-        //  attributes (e.g., Domain) is unspecified.
+        //  attributes (e.g., Domain) is unspecified.
-        $domain = $param['domain'];
+        $domain = $param['domain'];
-        if (strpos($domain, '.') === false) $domain .= '.local';
+        if (strpos($domain, '.') === false) $domain .= '.local';
-        $request_path = $param['path'];
+        $request_path = $param['path'];
-        if ($request_path == '') $request_path = '/';
+        if ($request_path == '') $request_path = '/';
-        $request_secure = $param['secure'];
+        $request_secure = $param['secure'];
-        $now = time();
+        $now = time();
-        $matched_cookies = array();
+        $matched_cookies = array();
-        // domain - find matching domains
+        // domain - find matching domains
-        $this->debug('Finding matching domains for '.$domain, __FILE__, __LINE__);
+        $this->debug('Finding matching domains for '.$domain, __FILE__, __LINE__);
-        while (strpos($domain, '.') !== false) {
+        while (strpos($domain, '.') !== false) {
-            if (isset($this->cookies[$domain])) {
+            if (isset($this->cookies[$domain])) {
-                $this->debug(' domain match found: '.$domain);
+                $this->debug(' domain match found: '.$domain);
-                $cookies =& $this->cookies[$domain];
+                $cookies =& $this->cookies[$domain];
-            } else {
+            } else {
-                $domain = $this->_reduce_domain($domain);
+                $domain = $this->_reduce_domain($domain);
-                continue;
+                continue;
-            }
+            }
-            // paths - find matching paths starting from most specific
+            // paths - find matching paths starting from most specific
-            $this->debug('  - Finding matching paths for '.$request_path);
+            $this->debug('  - Finding matching paths for '.$request_path);
-            $paths = array_keys($cookies);
+            $paths = array_keys($cookies);
-            usort($paths, array($this, '_cmp_length'));
+            usort($paths, array($this, '_cmp_length'));
-            foreach ($paths as $path) {
+            foreach ($paths as $path) {
-                // continue to next cookie if request path does not path-match cookie path
+                // continue to next cookie if request path does not path-match cookie path
-                if (!$this->_path_match($request_path, $path)) continue;
+                if (!$this->_path_match($request_path, $path)) continue;
-                // loop through cookie names
+                // loop through cookie names
-                $this->debug('     path match found: '.$path);
+                $this->debug('     path match found: '.$path);
-                foreach ($cookies[$path] as $name => $values) {
+                foreach ($cookies[$path] as $name => $values) {
-                    // if this cookie is secure but request isn't, continue to next cookie
+                    // if this cookie is secure but request isn't, continue to next cookie
-                    if ($values[1] && !$request_secure) continue;
+                    if ($values[1] && !$request_secure) continue;
-                    // if cookie is not a session cookie and has expired, continue to next cookie
+                    // if cookie is not a session cookie and has expired, continue to next cookie
-                    if (is_int($values[2]) && ($values[2] < $now)) continue;
+                    if (is_int($values[2]) && ($values[2] < $now)) continue;
-                    // cookie matches request
+                    // cookie matches request
-                    $this->debug('      cookie match: '.$name.'='.$values[0]);
+                    $this->debug('      cookie match: '.$name.'='.$values[0]);
-                    $matched_cookies[] = $name.'='.$values[0];
+                    $matched_cookies[] = $name.'='.$values[0];
-                }
+                }
-            }
+            }
-            $domain = $this->_reduce_domain($domain);
+            $domain = $this->_reduce_domain($domain);
-        }
+        }
-        // return cookies
+        // return cookies
-        return implode('; ', $matched_cookies);
+        return implode('; ', $matched_cookies);
-    }
+    }
-    /**
+    /**
-    * Parse Set-Cookie values.
+    * Parse Set-Cookie values.
-    *
+    *
-    * Only use this method if you cannot use extract_cookies(), for example, if you want to use
+    * Only use this method if you cannot use extract_cookies(), for example, if you want to use
-    * this cookie jar class without using the response class.
+    * this cookie jar class without using the response class.
-    *
+    *
-    * @param array $set_cookies array holding 1 or more "Set-Cookie" header values
+    * @param array $set_cookies array holding 1 or more "Set-Cookie" header values
-    * @param array $param associative array containing 'host', 'path' keys
+    * @param array $param associative array containing 'host', 'path' keys
-    * @return void
+    * @return void
-    * @see extract_cookies()
+    * @see extract_cookies()
-    */
+    */
-    public function storeCookies($url, $set_cookies)
+    public function storeCookies($url, $set_cookies)
-    {
+    {
-        if (count($set_cookies) == 0) return;
+        if (count($set_cookies) == 0) return;
-                $param = @parse_url($url);
+                $param = @parse_url($url);
-                if (!is_array($param) || !isset($param['host'])) return;
+                if (!is_array($param) || !isset($param['host'])) return;
-        $request_host = $param['host'];
+        $request_host = $param['host'];
-        if (strpos($request_host, '.') === false) $request_host .= '.local';
+        if (strpos($request_host, '.') === false) $request_host .= '.local';
-        $request_path = @$param['path'];
+        $request_path = @$param['path'];
-        if ($request_path == '') $request_path = '/';
+        if ($request_path == '') $request_path = '/';
-        //
+        //
-        // loop through set-cookie headers
+        // loop through set-cookie headers
-        //
+        //
-        foreach ($set_cookies as $set_cookie) {
+        foreach ($set_cookies as $set_cookie) {
-            $this->debug('Parsing: '.$set_cookie);
+            $this->debug('Parsing: '.$set_cookie);
-            // temporary cookie store (before adding to jar)
+            // temporary cookie store (before adding to jar)
-            $tmp_cookie = array();
+            $tmp_cookie = array();
-            $param = explode(';', $set_cookie);
+            $param = explode(';', $set_cookie);
-            // loop through params
+            // loop through params
-            for ($x=0; $x<count($param); $x++) {
+            for ($x=0; $x<count($param); $x++) {
-                $key_val = explode('=', $param[$x], 2);
+                $key_val = explode('=', $param[$x], 2);
-                if (count($key_val) != 2) {
+                if (count($key_val) != 2) {
-                    // if the first param isn't a name=value pair, continue to the next set-cookie
+                    // if the first param isn't a name=value pair, continue to the next set-cookie
-                    // header
+                    // header
-                    if ($x == 0) continue 2;
+                    if ($x == 0) continue 2;
-                    // check for secure flag
+                    // check for secure flag
-                    if (strtolower(trim($key_val[0])) == 'secure') $tmp_cookie['secure'] = true;
+                    if (strtolower(trim($key_val[0])) == 'secure') $tmp_cookie['secure'] = true;
-                    // continue to next param
+                    // continue to next param
-                    continue;
+                    continue;
-                }
+                }
-                list($key, $val) = array_map('trim', $key_val);
+                list($key, $val) = array_map('trim', $key_val);
-                // first name=value pair is the cookie name and value
+                // first name=value pair is the cookie name and value
-                // the name and value are stored under 'name' and 'value' to avoid conflicts
+                // the name and value are stored under 'name' and 'value' to avoid conflicts
-                // with later parameters.
+                // with later parameters.
-                if ($x == 0) {
+                if ($x == 0) {
-                    $tmp_cookie = array('name'=>$key, 'value'=>$val);
+                    $tmp_cookie = array('name'=>$key, 'value'=>$val);
-                    continue;
+                    continue;
-                }
+                }
-                $key = strtolower($key);
+                $key = strtolower($key);
-                if (in_array($key, array('expires', 'path', 'domain', 'secure'))) {
+                if (in_array($key, array('expires', 'path', 'domain', 'secure'))) {
-                    $tmp_cookie[$key] = $val;
+                    $tmp_cookie[$key] = $val;
-                }
+                }
-            }
+            }
-            //
+            //
-            // set cookie
+            // set cookie
-            //
+            //
-            // check domain
+            // check domain
-            if (isset($tmp_cookie['domain']) && ($tmp_cookie['domain'] != $request_host) &&
+            if (isset($tmp_cookie['domain']) && ($tmp_cookie['domain'] != $request_host) &&
-                    ($tmp_cookie['domain'] != ".$request_host")) {
+                    ($tmp_cookie['domain'] != ".$request_host")) {
-                $domain = $tmp_cookie['domain'];
+                $domain = $tmp_cookie['domain'];
-                if ((strpos($domain, '.') === false) && ($domain != 'local')) {
+                if ((strpos($domain, '.') === false) && ($domain != 'local')) {
-                    $this->debug(' - domain "'.$domain.'" has no dot and is not a local domain');
+                    $this->debug(' - domain "'.$domain.'" has no dot and is not a local domain');
-                    continue;
+                    continue;
-                }
+                }
-                if (preg_match('/\.[0-9]+$/', $domain)) {
+                if (preg_match('/\.[0-9]+$/', $domain)) {
-                    $this->debug(' - domain "'.$domain.'" appears to be an ip address');
+                    $this->debug(' - domain "'.$domain.'" appears to be an ip address');
-                    continue;
+                    continue;
-                }
+                }
-                if (substr($domain, 0, 1) != '.') $domain = ".$domain";
+                if (substr($domain, 0, 1) != '.') $domain = ".$domain";
-                if (!$this->_domain_match($request_host, $domain)) {
+                if (!$this->_domain_match($request_host, $domain)) {
-                    $this->debug(' - request host "'.$request_host.'" does not domain-match "'.$domain.'"');
+                    $this->debug(' - request host "'.$request_host.'" does not domain-match "'.$domain.'"');
-                    continue;
+                    continue;
-                }
+                }
-            } else {
+            } else {
-                // if domain is not specified in the set-cookie header, domain will default to
+                // if domain is not specified in the set-cookie header, domain will default to
-                // the request host
+                // the request host
-                $domain = $request_host;
+                $domain = $request_host;
-            }
+            }
-            // check path
+            // check path
-            if (isset($tmp_cookie['path']) && ($tmp_cookie['path'] != '')) {
+            if (isset($tmp_cookie['path']) && ($tmp_cookie['path'] != '')) {
-                $path = urldecode($tmp_cookie['path']);
+                $path = urldecode($tmp_cookie['path']);
-                if (!$this->_path_match($request_path, $path)) {
+                if (!$this->_path_match($request_path, $path)) {
-                    $this->debug(' - request path "'.$request_path.'" does not path-match "'.$path.'"');
+                    $this->debug(' - request path "'.$request_path.'" does not path-match "'.$path.'"');
-                    continue;
+                    continue;
-                }
+                }
-            } else {
+            } else {
-                $path = $request_path;
+                $path = $request_path;
-                $path = substr($path, 0, strrpos($path, '/'));
+                $path = substr($path, 0, strrpos($path, '/'));
-                if ($path == '') $path = '/';
+                if ($path == '') $path = '/';
-            }
+            }
-            // check if secure
+            // check if secure
-            $secure = (isset($tmp_cookie['secure'])) ? true : false;
+            $secure = (isset($tmp_cookie['secure'])) ? true : false;
-            // check expiry
+            // check expiry
-            if (isset($tmp_cookie['expires'])) {
+            if (isset($tmp_cookie['expires'])) {
-                if (($expires = strtotime($tmp_cookie['expires'])) < 0) {
+                if (($expires = strtotime($tmp_cookie['expires'])) < 0) {
-                    $expires = null;
+                    $expires = null;
-                }
+                }
-            } else {
+            } else {
-                $expires = null;
+                $expires = null;
-            }
+            }
-            // set cookie
+            // set cookie
-            $this->set_cookie($domain, $path, $tmp_cookie['name'], $tmp_cookie['value'], $secure, $expires);
+            $this->set_cookie($domain, $path, $tmp_cookie['name'], $tmp_cookie['value'], $secure, $expires);
-        }
+        }
-    }
+    }
-        
+        
-        // return array of set-cookie values extracted from HTTP response headers (string $h)
+        // return array of set-cookie values extracted from HTTP response headers (string $h)
-        public function extractCookies($h) {
+        public function extractCookies($h) {
-        $x = 0;
+        $x = 0;
-        $lines = 0;
+        $lines = 0;
-        $headers = array();
+        $headers = array();
-        $last_match = false;
+        $last_match = false;
-                $h = explode("\n", $h);
+                $h = explode("\n", $h);
-        foreach ($h as $line) {
+        foreach ($h as $line) {
-                        $line = rtrim($line);
+                        $line = rtrim($line);
-            $lines++;
+            $lines++;
-            $trimmed_line = trim($line);
+            $trimmed_line = trim($line);
-            if (isset($line_last)) {
+            if (isset($line_last)) {
-                // check if we have \r\n\r\n (indicating the end of headers)
+                // check if we have \r\n\r\n (indicating the end of headers)
-                // some servers will not use CRLF (\r\n), so we make CR (\r) optional.
+                // some servers will not use CRLF (\r\n), so we make CR (\r) optional.
-                // if (preg_match('/\015?\012\015?\012/', $line_last.$line)) {
+                // if (preg_match('/\015?\012\015?\012/', $line_last.$line)) {
-                //     break;
+                //     break;
-                // }
+                // }
-                // As an alternative, we can check if the current trimmed line is empty
+                // As an alternative, we can check if the current trimmed line is empty
-                if ($trimmed_line == '') {
+                if ($trimmed_line == '') {
-                    break;
+                    break;
-                }
+                }
-                // check for continuation line...
+                // check for continuation line...
-                // RFC 2616 Section 2.2 "Basic Rules":
+                // RFC 2616 Section 2.2 "Basic Rules":
-                // HTTP/1.1 header field values can be folded onto multiple lines if the
+                // HTTP/1.1 header field values can be folded onto multiple lines if the
-                // continuation line begins with a space or horizontal tab. All linear
+                // continuation line begins with a space or horizontal tab. All linear
-                // white space, including folding, has the same semantics as SP. A
+                // white space, including folding, has the same semantics as SP. A
-                // recipient MAY replace any linear white space with a single SP before
+                // recipient MAY replace any linear white space with a single SP before
-                // interpreting the field value or forwarding the message downstream.
+                // interpreting the field value or forwarding the message downstream.
-                if ($last_match && preg_match('/^\s+(.*)/', $line, $match)) {
+                if ($last_match && preg_match('/^\s+(.*)/', $line, $match)) {
-                    // append to previous header value
+                    // append to previous header value
-                    $headers[$x-1] .= ' '.rtrim($match[1]);
+                    $headers[$x-1] .= ' '.rtrim($match[1]);
-                    continue;
+                    continue;
-                }
+                }
-            }
+            }
-            $line_last = $line;
+            $line_last = $line;
-            // split header name and value
+            // split header name and value
-            if (preg_match('/^Set-Cookie\s*:\s*(.*)/i', $line, $match)) {
+            if (preg_match('/^Set-Cookie\s*:\s*(.*)/i', $line, $match)) {
-                $headers[$x++] = rtrim($match[1]);
+                $headers[$x++] = rtrim($match[1]);
-                $last_match = true;
+                $last_match = true;
-            } else {
+            } else {
-                $last_match = false;
+                $last_match = false;
-            }
+            }
-        }
+        }
-        return $headers;
+        return $headers;
-        }
+        }
-    /**
+    /**
-    * Set Cookie
+    * Set Cookie
-    * @param string $domain
+    * @param string $domain
-    * @param string $path
+    * @param string $path
-    * @param string $name cookie name
+    * @param string $name cookie name
-    * @param string $value cookie value
+    * @param string $value cookie value
-    * @param bool $secure
+    * @param bool $secure
-    * @param int $expires expiry time (null if session cookie, <= 0 will delete cookie)
+    * @param int $expires expiry time (null if session cookie, <= 0 will delete cookie)
-    * @return void
+    * @return void
-    */
+    */
-    function set_cookie($domain, $path, $name, $value, $secure=false, $expires=null)
+    function set_cookie($domain, $path, $name, $value, $secure=false, $expires=null)
-    {
+    {
-        if ($domain == '') return;
+        if ($domain == '') return;
-        if ($path == '') return;
+        if ($path == '') return;
-        if ($name == '') return;
+        if ($name == '') return;
-        // check if cookie needs to go
+        // check if cookie needs to go
-        if (isset($expires) && ($expires <= 0)) {
+        if (isset($expires) && ($expires <= 0)) {
-            if (isset($this->cookies[$domain][$path][$name])) unset($this->cookies[$domain][$path][$name]);
+            if (isset($this->cookies[$domain][$path][$name])) unset($this->cookies[$domain][$path][$name]);
-            return;
+            return;
-        }
+        }
-        if ($value == '') return;
+        if ($value == '') return;
-        $this->cookies[$domain][$path][$name] = array($value, $secure, $expires);
+        $this->cookies[$domain][$path][$name] = array($value, $secure, $expires);
-        return;
+        return;
-    }
+    }
-    /**
+    /**
-    * Clear cookies - [domain [,path [,name]]] - call method with no arguments to clear all cookies.
+    * Clear cookies - [domain [,path [,name]]] - call method with no arguments to clear all cookies.
-    * @param string $domain
+    * @param string $domain
-    * @param string $path
+    * @param string $path
-    * @param string $name
+    * @param string $name
-    * @return void
+    * @return void
-    */
+    */
-    function clear($domain=null, $path=null, $name=null)
+    function clear($domain=null, $path=null, $name=null)
-    {
+    {
-        if (!isset($domain)) {
+        if (!isset($domain)) {
-            $this->cookies = array();
+            $this->cookies = array();
-        } elseif (!isset($path)) {
+        } elseif (!isset($path)) {
-            if (isset($this->cookies[$domain])) unset($this->cookies[$domain]);
+            if (isset($this->cookies[$domain])) unset($this->cookies[$domain]);
-        } elseif (!isset($name)) {
+        } elseif (!isset($name)) {
-            if (isset($this->cookies[$domain][$path])) unset($this->cookies[$domain][$path]);
+            if (isset($this->cookies[$domain][$path])) unset($this->cookies[$domain][$path]);
-        } elseif (isset($name)) {
+        } elseif (isset($name)) {
-            if (isset($this->cookies[$domain][$path][$name])) unset($this->cookies[$domain][$path][$name]);
+            if (isset($this->cookies[$domain][$path][$name])) unset($this->cookies[$domain][$path][$name]);
-        }
+        }
-    }
+    }
-    /**
+    /**
-    * Compare string length - used for sorting
+    * Compare string length - used for sorting
-    * @access private
+    * @access private
-    * @return int
+    * @return int
-    */
+    */
-    function _cmp_length($a, $b)
+    function _cmp_length($a, $b)
-    {
+    {
-        $la = strlen($a); $lb = strlen($b);
+        $la = strlen($a); $lb = strlen($b);
-        if ($la == $lb) return 0;
+        if ($la == $lb) return 0;
-        return ($la > $lb) ? -1 : 1;
+        return ($la > $lb) ? -1 : 1;
-    }
+    }
-    /**
+    /**
-    * Reduce domain
+    * Reduce domain
-    * @param string $domain
+    * @param string $domain
-    * @return string
+    * @return string
-    * @access private
+    * @access private
-    */
+    */
-    function _reduce_domain($domain)
+    function _reduce_domain($domain)
-    {
+    {
-        if ($domain == '') return '';
+        if ($domain == '') return '';
-        if (substr($domain, 0, 1) == '.') return substr($domain, 1);
+        if (substr($domain, 0, 1) == '.') return substr($domain, 1);
-        return substr($domain, strpos($domain, '.'));
+        return substr($domain, strpos($domain, '.'));
-    }
+    }
-    /**
+    /**
-    * Path match - check if path1 path-matches path2
+    * Path match - check if path1 path-matches path2
-    *
+    *
-    * From RFC 2965: 
+    * From RFC 2965: 
-    *   <i>For two strings that represent paths, P1 and P2, P1 path-matches P2
+    *   <i>For two strings that represent paths, P1 and P2, P1 path-matches P2
-    *   if P2 is a prefix of P1 (including the case where P1 and P2 string-
+    *   if P2 is a prefix of P1 (including the case where P1 and P2 string-
-    *   compare equal).  Thus, the string /tec/waldo path-matches /tec.</i>
+    *   compare equal).  Thus, the string /tec/waldo path-matches /tec.</i>
-    * @param string $path1
+    * @param string $path1
-    * @param string $path2
+    * @param string $path2
-    * @return bool
+    * @return bool
-    * @access private
+    * @access private
-    */
+    */
-    function _path_match($path1, $path2)
+    function _path_match($path1, $path2)
-    {
+    {
-        return (substr($path1, 0, strlen($path2)) == $path2);
+        return (substr($path1, 0, strlen($path2)) == $path2);
-    }
+    }
-    /**
+    /**
-    * Domain match - check if domain1 domain-matches domain2
+    * Domain match - check if domain1 domain-matches domain2
-    *
+    *
-    * A few extracts from RFC 2965: 
+    * A few extracts from RFC 2965: 
-    *  -  A Set-Cookie2 from request-host y.x.foo.com for Domain=.foo.com
+    *  -  A Set-Cookie2 from request-host y.x.foo.com for Domain=.foo.com
-    *     would be rejected, because H is y.x and contains a dot.
+    *     would be rejected, because H is y.x and contains a dot.
-    *
+    *
-    *  -  A Set-Cookie2 from request-host x.foo.com for Domain=.foo.com
+    *  -  A Set-Cookie2 from request-host x.foo.com for Domain=.foo.com
-    *     would be accepted.
+    *     would be accepted.
-    *
+    *
-    *  -  A Set-Cookie2 with Domain=.com or Domain=.com., will always be
+    *  -  A Set-Cookie2 with Domain=.com or Domain=.com., will always be
-    *     rejected, because there is no embedded dot.
+    *     rejected, because there is no embedded dot.
-    *
+    *
-    *  -  A Set-Cookie2 from request-host example for Domain=.local will
+    *  -  A Set-Cookie2 from request-host example for Domain=.local will
-    *     be accepted, because the effective host name for the request-
+    *     be accepted, because the effective host name for the request-
-    *     host is example.local, and example.local domain-matches .local.
+    *     host is example.local, and example.local domain-matches .local.
-    *
+    *
-    * I'm ignoring the first point for now (must check to see how other browsers handle
+    * I'm ignoring the first point for now (must check to see how other browsers handle
-    * this rule for Set-Cookie headers)
+    * this rule for Set-Cookie headers)
-    *
+    *
-    * @param string $domain1
+    * @param string $domain1
-    * @param string $domain2
+    * @param string $domain2
-    * @return bool
+    * @return bool
-    * @access private
+    * @access private
-    */
+    */
-    function _domain_match($domain1, $domain2)
+    function _domain_match($domain1, $domain2)
-    {
+    {
-        $domain1 = strtolower($domain1);
+        $domain1 = strtolower($domain1);
-        $domain2 = strtolower($domain2);
+        $domain2 = strtolower($domain2);
-        while (strpos($domain1, '.') !== false) {
+        while (strpos($domain1, '.') !== false) {
-            if ($domain1 == $domain2) return true;
+            if ($domain1 == $domain2) return true;
-            $domain1 = $this->_reduce_domain($domain1);
+            $domain1 = $this->_reduce_domain($domain1);
-            continue;
+            continue;
-        }
+        }
-        return false;
+        return false;
-    }
+    }
-}
+}
+\ No newline at end of file
-?>
-\ No newline at end of file
diff --git a/inc/3rdparty/libraries/humble-http-agent/HumbleHttpAgent.php b/inc/3rdparty/libraries/humble-http-agent/HumbleHttpAgent.php
index e4f1b3b3..963f0c05 100644
--- a/inc/3rdparty/libraries/humble-http-agent/HumbleHttpAgent.php
+++ b/inc/3rdparty/libraries/humble-http-agent/HumbleHttpAgent.php
@@ -1,779 +1,810 @@
-<?php
+<?php
-/**
+/**
- * Humble HTTP Agent
+ * Humble HTTP Agent
- * 
+ * 
- * This class is designed to take advantage of parallel HTTP requests
+ * This class is designed to take advantage of parallel HTTP requests
- * offered by PHP's PECL HTTP extension or the curl_multi_* functions. 
+ * offered by PHP's PECL HTTP extension or the curl_multi_* functions. 
- * For environments which do not have these options, it reverts to standard sequential 
+ * For environments which do not have these options, it reverts to standard sequential 
- * requests (using file_get_contents())
+ * requests (using file_get_contents())
- * 
+ * 
- * @version 1.1
+ * @version 1.4
- * @date 2012-08-20
+ * @date 2013-05-10
- * @see http://php.net/HttpRequestPool
+ * @see http://php.net/HttpRequestPool
- * @author Keyvan Minoukadeh
+ * @author Keyvan Minoukadeh
- * @copyright 2011-2012 Keyvan Minoukadeh
+ * @copyright 2011-2013 Keyvan Minoukadeh
- * @license http://www.gnu.org/licenses/agpl-3.0.html AGPL v3
+ * @license http://www.gnu.org/licenses/agpl-3.0.html AGPL v3
- */
+ */
-class HumbleHttpAgent
+class HumbleHttpAgent
-{
+{
-        const METHOD_REQUEST_POOL = 1;
+        const METHOD_REQUEST_POOL = 1;
-        const METHOD_CURL_MULTI = 2;
+        const METHOD_CURL_MULTI = 2;
-        const METHOD_FILE_GET_CONTENTS = 4;
+        const METHOD_FILE_GET_CONTENTS = 4;
-        //const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1';
+        //const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1';
-        const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2';
+        const UA_BROWSER = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2';
-        const UA_PHP = 'PHP/5.2';
+        const UA_PHP = 'PHP/5.4';
-        const REF_GOOGLE = 'http://www.google.co.uk/url?sa=t&source=web&cd=1';
+        const REF_GOOGLE = 'http://www.google.co.uk/url?sa=t&source=web&cd=1';
-        
+        
-        protected $requests = array();
+        protected $requests = array();
-        protected $redirectQueue = array();
+        protected $redirectQueue = array();
-        protected $requestOptions;
+        protected $requestOptions;
-        protected $maxParallelRequests = 5;
+        protected $maxParallelRequests = 5;
-        protected $cache = null; //TODO
+        protected $cache = null; //TODO
-        protected $httpContext;
+        protected $httpContext;
-        protected $minimiseMemoryUse = false; //TODO
+        protected $minimiseMemoryUse = false; //TODO
-        protected $method;
+        protected $method;
-        protected $cookieJar;
+        protected $cookieJar;
-        public $debug = false;
+        public $debug = false;
-        public $debugVerbose = false;
+        public $debugVerbose = false;
-        public $rewriteHashbangFragment = true; // see http://code.google.com/web/ajaxcrawling/docs/specification.html
+        public $rewriteHashbangFragment = true; // see http://code.google.com/web/ajaxcrawling/docs/specification.html
-        public $maxRedirects = 5;
+        public $maxRedirects = 5;
-        public $userAgentMap = array();
+        public $userAgentMap = array();
-        public $rewriteUrls = array();
+        public $rewriteUrls = array();
-        public $userAgentDefault;
+        public $userAgentDefault;
-        public $referer;
+        public $referer;
-        //public $userAgent = 'Mozilla/5.0';
+        //public $userAgent = 'Mozilla/5.0';
-        
+        
-        // Prevent certain file/mime types
+        // Prevent certain file/mime types
-        // HTTP responses which match these content types will
+        // HTTP responses which match these content types will
-        // be returned without body.
+        // be returned without body.
-        public $headerOnlyTypes = array();
+        public $headerOnlyTypes = array();
-        // URLs ending with one of these extensions will
+        // URLs ending with one of these extensions will
-        // prompt Humble HTTP Agent to send a HEAD request first
+        // prompt Humble HTTP Agent to send a HEAD request first
-        // to see if returned content type matches $headerOnlyTypes.
+        // to see if returned content type matches $headerOnlyTypes.
-        public $headerOnlyClues = array('pdf','mp3','zip','exe','gif','gzip','gz','jpeg','jpg','mpg','mpeg','png','ppt','mov');
+        public $headerOnlyClues = array('pdf','mp3','zip','exe','gif','gzip','gz','jpeg','jpg','mpg','mpeg','png','ppt','mov');
-        // AJAX triggers to search for.
+        // AJAX triggers to search for.
-        // for AJAX sites, e.g. Blogger with its dynamic views templates.
+        // for AJAX sites, e.g. Blogger with its dynamic views templates.
-        public $ajaxTriggers = array("<meta name='fragment' content='!'",'<meta name="fragment" content="!"',"<meta content='!' name='fragment'",'<meta content="!" name="fragment"');
+        public $ajaxTriggers = array("<meta name='fragment' content='!'",'<meta name="fragment" content="!"',"<meta content='!' name='fragment'",'<meta content="!" name="fragment"');
-        
+        
-        //TODO: set max file size
+        //TODO: set max file size
-        //TODO: normalise headers
+        //TODO: normalise headers
-        
+        
-        function __construct($requestOptions=null, $method=null) {
+        function __construct($requestOptions=null, $method=null) {
-                $this->userAgentDefault = self::UA_BROWSER;
+                $this->userAgentDefault = self::UA_BROWSER;
-                $this->referer = self::REF_GOOGLE;
+                $this->referer = self::REF_GOOGLE;
-                // set the request method
+                // set the request method
-                if (in_array($method, array(1,2,4))) {
+                if (in_array($method, array(1,2,4))) {
-                        $this->method = $method;
+                        $this->method = $method;
-                } else {
+                } else {
-                        if (class_exists('HttpRequestPool')) {
+                        if (class_exists('HttpRequestPool')) {
-                                $this->method = self::METHOD_REQUEST_POOL;
+                                $this->method = self::METHOD_REQUEST_POOL;
-                        } elseif (function_exists('curl_multi_init')) {
+                        } elseif (function_exists('curl_multi_init')) {
-                                $this->method = self::METHOD_CURL_MULTI;
+                                $this->method = self::METHOD_CURL_MULTI;
-                        } else {
+                        } else {
-                                $this->method = self::METHOD_FILE_GET_CONTENTS;
+                                $this->method = self::METHOD_FILE_GET_CONTENTS;
-                        }
+                        }
-                }
+                }
-                if ($this->method == self::METHOD_CURL_MULTI) {
+                if ($this->method == self::METHOD_CURL_MULTI) {
-                        require_once(dirname(__FILE__).'/RollingCurl.php');
+                        require_once(dirname(__FILE__).'/RollingCurl.php');
-                }
+                }
-                // create cookie jar
+                // create cookie jar
-                $this->cookieJar = new CookieJar();
+                $this->cookieJar = new CookieJar();
-                // set request options (redirect must be 0)
+                // set request options (redirect must be 0)
-                $this->requestOptions = array(
+                $this->requestOptions = array(
-                        'timeout' => 15,
+                        'timeout' => 15,
-                        'redirect' => 0 // we handle redirects manually so we can rewrite the new hashbang URLs that are creeping up over the web
+                        'connecttimeout' => 15,
-                        // TODO: test onprogress?
+                        'dns_cache_timeout' => 300,
-                );
+                        'redirect' => 0 // we handle redirects manually so we can rewrite the new hashbang URLs that are creeping up over the web
-                if (is_array($requestOptions)) {
+                        // TODO: test onprogress?
-                        $this->requestOptions = array_merge($this->requestOptions, $requestOptions);
+                );
-                }
+                if (is_array($requestOptions)) {
-                $this->httpContext = array(
+                        $this->requestOptions = array_merge($this->requestOptions, $requestOptions);
-                        'http' => array(
+                }
-                                'ignore_errors' => true,
+                $this->httpContext = array(
-                                'timeout' => $this->requestOptions['timeout'],
+                        'http' => array(
-                                'max_redirects' => $this->requestOptions['redirect'],
+                                'ignore_errors' => true,
-                                'header' => "Accept: */*\r\n"
+                                'timeout' => $this->requestOptions['timeout'],
-                                )
+                                'max_redirects' => $this->requestOptions['redirect'],
-                        );
+                                'header' => "Accept: */*\r\n"
-        }
+                                )
-        
+                        );
-        protected function debug($msg) {
+        }
-                if ($this->debug) {
+        
-                        $mem = round(memory_get_usage()/1024, 2);
+        protected function debug($msg) {
-                        $memPeak = round(memory_get_peak_usage()/1024, 2);
+                if ($this->debug) {
-                        echo '* ',$msg;
+                        $mem = round(memory_get_usage()/1024, 2);
-                        if ($this->debugVerbose) echo ' - mem used: ',$mem," (peak: $memPeak)";
+                        $memPeak = round(memory_get_peak_usage()/1024, 2);
-                        echo "\n";
+                        echo '* ',$msg;
-                        ob_flush();
+                        if ($this->debugVerbose) echo ' - mem used: ',$mem," (peak: $memPeak)";
-                        flush();
+                        echo "\n";
-                }
+                        ob_flush();
-        }
+                        flush();
-        
+                }
-        protected function getUserAgent($url, $asArray=false) {
+        }
-                $host = @parse_url($url, PHP_URL_HOST);
+        
-                if (strtolower(substr($host, 0, 4)) == 'www.') {
+        protected function getUserAgent($url, $asArray=false) {
-                        $host = substr($host, 4);
+                $host = @parse_url($url, PHP_URL_HOST);
-                }
+                if (strtolower(substr($host, 0, 4)) == 'www.') {
-                if ($host) {
+                        $host = substr($host, 4);
-                        $try = array($host);
+                }
-                        $split = explode('.', $host);
+                if ($host) {
-                        if (count($split) > 1) {
+                        $try = array($host);
-                                array_shift($split);
+                        $split = explode('.', $host);
-                                $try[] = '.'.implode('.', $split);
+                        if (count($split) > 1) {
-                        }
+                                array_shift($split);
-                        foreach ($try as $h) {
+                                $try[] = '.'.implode('.', $split);
-                                if (isset($this->userAgentMap[$h])) {
+                        }
-                                        $ua = $this->userAgentMap[$h];
+                        foreach ($try as $h) {
-                                        break;
+                                if (isset($this->userAgentMap[$h])) {
-                                }
+                                        $ua = $this->userAgentMap[$h];
-                        }
+                                        break;
-                }
+                                }
-                if (!isset($ua)) $ua = $this->userAgentDefault;
+                        }
-                if ($asArray) {
+                }
-                        return array('User-Agent' => $ua);
+                if (!isset($ua)) $ua = $this->userAgentDefault;
-                } else {
+                if ($asArray) {
-                        return 'User-Agent: '.$ua;
+                        return array('User-Agent' => $ua);
-                }
+                } else {
-        }
+                        return 'User-Agent: '.$ua;
-        
+                }
-        public function rewriteHashbangFragment($url) {
+        }
-                // return $url if there's no '#!'
+        
-                if (strpos($url, '#!') === false) return $url;
+        public function rewriteHashbangFragment($url) {
-                // split $url and rewrite
+                // return $url if there's no '#!'
-                // TODO: is SimplePie_IRI included?
+                if (strpos($url, '#!') === false) return $url;
-                $iri = new SimplePie_IRI($url);
+                // split $url and rewrite
-                $fragment = substr($iri->fragment, 1); // strip '!'
+                // TODO: is SimplePie_IRI included?
-                $iri->fragment = null;
+                $iri = new SimplePie_IRI($url);
-                if (isset($iri->query)) {
+                $fragment = substr($iri->fragment, 1); // strip '!'
-                        parse_str($iri->query, $query);
+                $iri->fragment = null;
-                } else {
+                if (isset($iri->query)) {
-                        $query = array();
+                        parse_str($iri->query, $query);
-                }
+                } else {
-                $query['_escaped_fragment_'] = (string)$fragment;
+                        $query = array();
-                $iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites
+                }
-                return $iri->get_iri();
+                $query['_escaped_fragment_'] = (string)$fragment;
-        }
+                $iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites
-        
+                return $iri->get_iri();
-        public function getUglyURL($url, $html) {
+        }
-                if ($html == '') return false;
+        
-                $found = false;
+        public function getRedirectURLfromHTML($url, $html) {
-                foreach ($this->ajaxTriggers as $string) {
+                $redirect_url = $this->getMetaRefreshURL($url, $html);
-                        if (stripos($html, $string)) {
+                if (!$redirect_url) {
-                                $found = true;
+                        $redirect_url = $this->getUglyURL($url, $html);
-                                break;
+                }
-                        }
+                return $redirect_url;
-                }
+        }
-                if (!$found) return false;
+        
-                $iri = new SimplePie_IRI($url);
+        public function getMetaRefreshURL($url, $html) {
-                if (isset($iri->query)) {
+                if ($html == '') return false;
-                        parse_str($iri->query, $query);
+                // <meta HTTP-EQUIV="REFRESH" content="0; url=http://www.bernama.com/bernama/v6/newsindex.php?id=943513">
-                } else {
+                if (!preg_match('!<meta http-equiv=["\']?refresh["\']? content=["\']?[0-9];\s*url=["\']?([^"\'>]+)["\']*>!i', $html, $match)) {
-                        $query = array();
+                        return false;
-                }
+                }
-                $query['_escaped_fragment_'] = '';
+                $redirect_url = $match[1];
-                $iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites
+                if (preg_match('!^https?://!i', $redirect_url)) {
-                return $iri->get_iri();
+                        // already absolute
-        }
+                        $this->debug('Meta refresh redirect found (http-equiv="refresh"), new URL: '.$redirect_url);
-        
+                        return $redirect_url;
-        public function removeFragment($url) {
+                }
-                $pos = strpos($url, '#');
+                // absolutize redirect URL
-                if ($pos === false) {
+                $base = new SimplePie_IRI($url);
-                        return $url;
+                // remove '//' in URL path (causes URLs not to resolve properly)
-                } else {
+                if (isset($base->path)) $base->path = preg_replace('!//+!', '/', $base->path);
-                        return substr($url, 0, $pos);
+                if ($absolute = SimplePie_IRI::absolutize($base, $redirect_url)) {
-                }
+                        $this->debug('Meta refresh redirect found (http-equiv="refresh"), new URL: '.$absolute);
-        }
+                        return $absolute;
-        
+                }
-        public function rewriteUrls($url) {
+                return false;
-                foreach ($this->rewriteUrls as $find => $action) {
+        }       
-                        if (strpos($url, $find) !== false) {
+        
-                                if (is_array($action)) {
+        public function getUglyURL($url, $html) {
-                                        return strtr($url, $action);
+                if ($html == '') return false;
-                                }
+                $found = false;
-                        }
+                foreach ($this->ajaxTriggers as $string) {
-                }
+                        if (stripos($html, $string)) {
-                return $url;
+                                $found = true;
-        }
+                                break;
-        
+                        }
-        public function enableDebug($bool=true) {
+                }
-                $this->debug = (bool)$bool;
+                if (!$found) return false;
-        }
+                $iri = new SimplePie_IRI($url);
-        
+                if (isset($iri->query)) {
-        public function minimiseMemoryUse($bool = true) {
+                        parse_str($iri->query, $query);
-                $this->minimiseMemoryUse = $bool;
+                } else {
-        }
+                        $query = array();
-        
+                }
-        public function setMaxParallelRequests($max) {
+                $query['_escaped_fragment_'] = '';
-                $this->maxParallelRequests = $max;
+                $iri->query = str_replace('%2F', '/', http_build_query($query)); // needed for some sites
-        }
+                $ugly_url = $iri->get_iri();
-        
+                $this->debug('AJAX trigger (meta name="fragment" content="!") found, new URL: '.$ugly_url);
-        public function validateUrl($url) {
+                return $ugly_url;
-                $url = filter_var($url, FILTER_SANITIZE_URL);
+        }
-                $test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
+        
-                // deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2)
+        public function removeFragment($url) {
-                if ($test === false) {
+                $pos = strpos($url, '#');
-                        $test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
+                if ($pos === false) {
-                }
+                        return $url;
-                if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) {
+                } else {
-                        return $url;
+                        return substr($url, 0, $pos);
-                } else {
+                }
-                        return false;
+        }
-                }
+        
-        }
+        public function rewriteUrls($url) {
-        
+                foreach ($this->rewriteUrls as $find => $action) {
-        public function fetchAll(array $urls) {
+                        if (strpos($url, $find) !== false) {
-                $this->fetchAllOnce($urls, $isRedirect=false);
+                                if (is_array($action)) {
-                $redirects = 0;
+                                        return strtr($url, $action);
-                while (!empty($this->redirectQueue) && ++$redirects <= $this->maxRedirects) {
+                                }
-                        $this->debug("Following redirects #$redirects...");
+                        }
-                        $this->fetchAllOnce($this->redirectQueue, $isRedirect=true);
+                }
-                }
+                return $url;
-        }
+        }
-        
+        
-        // fetch all URLs without following redirects
+        public function enableDebug($bool=true) {
-        public function fetchAllOnce(array $urls, $isRedirect=false) {
+                $this->debug = (bool)$bool;
-                if (!$isRedirect) $urls = array_unique($urls);
+        }
-                if (empty($urls)) return;
+        
-                
+        public function minimiseMemoryUse($bool = true) {
-                //////////////////////////////////////////////////////
+                $this->minimiseMemoryUse = $bool;
-                // parallel (HttpRequestPool)
+        }
-                if ($this->method == self::METHOD_REQUEST_POOL) {
+        
-                        $this->debug('Starting parallel fetch (HttpRequestPool)');
+        public function setMaxParallelRequests($max) {
-                        try {
+                $this->maxParallelRequests = $max;
-                                while (count($urls) > 0) {
+        }
-                                        $this->debug('Processing set of '.min($this->maxParallelRequests, count($urls)));
+        
-                                        $subset = array_splice($urls, 0, $this->maxParallelRequests);
+        public function validateUrl($url) {
-                                        $pool = new HttpRequestPool();
+                $url = filter_var($url, FILTER_SANITIZE_URL);
-                                        foreach ($subset as $orig => $url) {
+                $test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
-                                                if (!$isRedirect) $orig = $url;
+                // deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2)
-                                                unset($this->redirectQueue[$orig]);
+                if ($test === false) {
-                                                $this->debug("...$url");
+                        $test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);
-                                                if (!$isRedirect && isset($this->requests[$url])) {
+                }
-                                                        $this->debug("......in memory");
+                if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) {
-                                                /*
+                        return $url;
-                                                } elseif ($this->isCached($url)) {
+                } else {
-                                                        $this->debug("......is cached");
+                        return false;
-                                                        if (!$this->minimiseMemoryUse) {
+                }
-                                                                $this->requests[$url] = $this->getCached($url);
+        }
-                                                        }
+        
-                                                */
+        public function fetchAll(array $urls) {
-                                                } else {
+                $this->fetchAllOnce($urls, $isRedirect=false);
-                                                        $this->debug("......adding to pool");
+                $redirects = 0;
-                                                        $req_url = $this->rewriteUrls($url);
+                while (!empty($this->redirectQueue) && ++$redirects <= $this->maxRedirects) {
-                                                        $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;
+                        $this->debug("Following redirects #$redirects...");
-                                                        $req_url = $this->removeFragment($req_url);
+                        $this->fetchAllOnce($this->redirectQueue, $isRedirect=true);
-                                                        if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) {
+                }
-                                                                $_meth = HttpRequest::METH_HEAD;
+        }
-                                                        } else {
+        
-                                                                $_meth = HttpRequest::METH_GET;
+        // fetch all URLs without following redirects
-                                                                unset($this->requests[$orig]['wrongGuess']);
+        public function fetchAllOnce(array $urls, $isRedirect=false) {
-                                                        }
+                if (!$isRedirect) $urls = array_unique($urls);
-                                                        $httpRequest = new HttpRequest($req_url, $_meth, $this->requestOptions);
+                if (empty($urls)) return;
-                                                        // send cookies, if we have any
+                
-                                                        if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
+                //////////////////////////////////////////////////////
-                                                                $this->debug("......sending cookies: $cookies");
+                // parallel (HttpRequestPool)
-                                                                $httpRequest->addHeaders(array('Cookie' => $cookies));
+                if ($this->method == self::METHOD_REQUEST_POOL) {
-                                                        }
+                        $this->debug('Starting parallel fetch (HttpRequestPool)');
-                                                        //$httpRequest->addHeaders(array('User-Agent' => $this->userAgent));
+                        try {
-                                                        $httpRequest->addHeaders($this->getUserAgent($req_url, true));
+                                while (count($urls) > 0) {
-                                                        // add referer for picky sites
+                                        $this->debug('Processing set of '.min($this->maxParallelRequests, count($urls)));
-                                                        $httpRequest->addheaders(array('Referer' => $this->referer));
+                                        $subset = array_splice($urls, 0, $this->maxParallelRequests);
-                                                        $this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest);
+                                        $pool = new HttpRequestPool();
-                                                        $this->requests[$orig]['original_url'] = $orig;
+                                        foreach ($subset as $orig => $url) {
-                                                        $pool->attach($httpRequest);
+                                                if (!$isRedirect) $orig = $url;
-                                                }
+                                                unset($this->redirectQueue[$orig]);
-                                        }
+                                                $this->debug("...$url");
-                                        // did we get anything into the pool?
+                                                if (!$isRedirect && isset($this->requests[$url])) {
-                                        if (count($pool) > 0) {
+                                                        $this->debug("......in memory");
-                                                $this->debug('Sending request...');
+                                                /*
-                                                try {
+                                                } elseif ($this->isCached($url)) {
-                                                        $pool->send();
+                                                        $this->debug("......is cached");
-                                                } catch (HttpRequestPoolException $e) {
+                                                        if (!$this->minimiseMemoryUse) {
-                                                        // do nothing
+                                                                $this->requests[$url] = $this->getCached($url);
-                                                }
+                                                        }
-                                                $this->debug('Received responses');
+                                                */
-                                                foreach($subset as $orig => $url) {
+                                                } else {
-                                                        if (!$isRedirect) $orig = $url;
+                                                        $this->debug("......adding to pool");
-                                                        $request = $this->requests[$orig]['httpRequest'];
+                                                        $req_url = $this->rewriteUrls($url);
-                                                        //$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader());
+                                                        $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;
-                                                        // getResponseHeader() doesn't return status line, so, for consistency...
+                                                        $req_url = $this->removeFragment($req_url);
-                                                        $this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size'));
+                                                        if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) {
-                                                        // check content type
+                                                                $_meth = HttpRequest::METH_HEAD;
-                                                        // TODO: use getResponseHeader('content-type') or getResponseInfo()
+                                                        } else {
-                                                        if ($this->headerOnlyType($this->requests[$orig]['headers'])) {
+                                                                $_meth = HttpRequest::METH_GET;
-                                                                $this->requests[$orig]['body'] = '';
+                                                                unset($this->requests[$orig]['wrongGuess']);
-                                                                $_header_only_type = true;
+                                                        }
-                                                                $this->debug('Header only type returned');
+                                                        $httpRequest = new HttpRequest($req_url, $_meth, $this->requestOptions);
-                                                        } else {
+                                                        // send cookies, if we have any
-                                                                $this->requests[$orig]['body'] = $request->getResponseBody();
+                                                        if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
-                                                                $_header_only_type = false;
+                                                                $this->debug("......sending cookies: $cookies");
-                                                        }
+                                                                $httpRequest->addHeaders(array('Cookie' => $cookies));
-                                                        $this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url');
+                                                        }
-                                                        $this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode();
+                                                        //$httpRequest->addHeaders(array('User-Agent' => $this->userAgent));
-                                                        // is redirect?
+                                                        $httpRequest->addHeaders($this->getUserAgent($req_url, true));
-                                                        if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) {
+                                                        // add referer for picky sites
-                                                                $redirectURL = $request->getResponseHeader('location');
+                                                        $httpRequest->addheaders(array('Referer' => $this->referer));
-                                                                if (!preg_match('!^https?://!i', $redirectURL)) {
+                                                        $this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest);
-                                                                        $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
+                                                        $this->requests[$orig]['original_url'] = $orig;
-                                                                }
+                                                        $pool->attach($httpRequest);
-                                                                if ($this->validateURL($redirectURL)) {
+                                                }
-                                                                        $this->debug('Redirect detected. Valid URL: '.$redirectURL);
+                                        }
-                                                                        // store any cookies
+                                        // did we get anything into the pool?
-                                                                        $cookies = $request->getResponseHeader('set-cookie');
+                                        if (count($pool) > 0) {
-                                                                        if ($cookies && !is_array($cookies)) $cookies = array($cookies);
+                                                $this->debug('Sending request...');
-                                                                        if ($cookies) $this->cookieJar->storeCookies($url, $cookies);
+                                                try {
-                                                                        $this->redirectQueue[$orig] = $redirectURL;
+                                                        $pool->send();
-                                                                } else {
+                                                } catch (HttpRequestPoolException $e) {
-                                                                        $this->debug('Redirect detected. Invalid URL: '.$redirectURL);
+                                                        // do nothing
-                                                                }
+                                                }
-                                                        } elseif (!$_header_only_type && $request->getMethod() === HttpRequest::METH_HEAD) {
+                                                $this->debug('Received responses');
-                                                                // the response content-type did not match our 'header only' types, 
+                                                foreach($subset as $orig => $url) {
-                                                                // but we'd issues a HEAD request because we assumed it would. So
+                                                        if (!$isRedirect) $orig = $url;
-                                                                // let's queue a proper GET request for this item...
+                                                        $request = $this->requests[$orig]['httpRequest'];
-                                                                $this->debug('Wrong guess at content-type, queing GET request');
+                                                        //$this->requests[$orig]['headers'] = $this->headersToString($request->getResponseHeader());
-                                                                $this->requests[$orig]['wrongGuess'] = true;
+                                                        // getResponseHeader() doesn't return status line, so, for consistency...
-                                                                $this->redirectQueue[$orig] = $this->requests[$orig]['effective_url'];
+                                                        $this->requests[$orig]['headers'] = substr($request->getRawResponseMessage(), 0, $request->getResponseInfo('header_size'));
-                                                        } elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {
+                                                        // check content type
-                                                                // check for <meta name='fragment' content='!'/>
+                                                        // TODO: use getResponseHeader('content-type') or getResponseInfo()
-                                                                // for AJAX sites, e.g. Blogger with its dynamic views templates.
+                                                        if ($this->headerOnlyType($this->requests[$orig]['headers'])) {
-                                                                // Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
+                                                                $this->requests[$orig]['body'] = '';
-                                                                if (isset($this->requests[$orig]['body'])) {
+                                                                $_header_only_type = true;
-                                                                        $redirectURL = $this->getUglyURL($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
+                                                                $this->debug('Header only type returned');
-                                                                        if ($redirectURL) {
+                                                        } else {
-                                                                                $this->debug('AJAX trigger (meta name="fragment" content="!") found. Queueing '.$redirectURL);
+                                                                $this->requests[$orig]['body'] = $request->getResponseBody();
-                                                                                $this->redirectQueue[$orig] = $redirectURL;
+                                                                $_header_only_type = false;
-                                                                        }
+                                                        }
-                                                                }
+                                                        $this->requests[$orig]['effective_url'] = $request->getResponseInfo('effective_url');
-                                                        }
+                                                        $this->requests[$orig]['status_code'] = $status_code = $request->getResponseCode();
-                                                        //die($url.' -multi- '.$request->getResponseInfo('effective_url'));
+                                                        // is redirect?
-                                                        $pool->detach($request);
+                                                        if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && $request->getResponseHeader('location')) {
-                                                        unset($this->requests[$orig]['httpRequest'], $request);
+                                                                $redirectURL = $request->getResponseHeader('location');
-                                                        /*
+                                                                if (!preg_match('!^https?://!i', $redirectURL)) {
-                                                        if ($this->minimiseMemoryUse) {
+                                                                        $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
-                                                                if ($this->cache($url)) {
+                                                                }
-                                                                        unset($this->requests[$url]);
+                                                                if ($this->validateURL($redirectURL)) {
-                                                                }
+                                                                        $this->debug('Redirect detected. Valid URL: '.$redirectURL);
-                                                        }
+                                                                        // store any cookies
-                                                        */
+                                                                        $cookies = $request->getResponseHeader('set-cookie');
-                                                }
+                                                                        if ($cookies && !is_array($cookies)) $cookies = array($cookies);
-                                        }
+                                                                        if ($cookies) $this->cookieJar->storeCookies($url, $cookies);
-                                }
+                                                                        $this->redirectQueue[$orig] = $redirectURL;
-                        } catch (HttpException $e) {
+                                                                } else {
-                                $this->debug($e);
+                                                                        $this->debug('Redirect detected. Invalid URL: '.$redirectURL);
-                                return false;
+                                                                }
-                        }
+                                                        } elseif (!$_header_only_type && $request->getMethod() === HttpRequest::METH_HEAD) {
-                }
+                                                                // the response content-type did not match our 'header only' types, 
-                
+                                                                // but we'd issues a HEAD request because we assumed it would. So
-                //////////////////////////////////////////////////////////
+                                                                // let's queue a proper GET request for this item...
-                // parallel (curl_multi_*)
+                                                                $this->debug('Wrong guess at content-type, queing GET request');
-                elseif ($this->method == self::METHOD_CURL_MULTI) {
+                                                                $this->requests[$orig]['wrongGuess'] = true;
-                        $this->debug('Starting parallel fetch (curl_multi_*)');
+                                                                $this->redirectQueue[$orig] = $this->requests[$orig]['effective_url'];
-                        while (count($urls) > 0) {
+                                                        } elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {
-                                $this->debug('Processing set of '.min($this->maxParallelRequests, count($urls)));
+                                                                // check for <meta name='fragment' content='!'/>
-                                $subset = array_splice($urls, 0, $this->maxParallelRequests);
+                                                                // for AJAX sites, e.g. Blogger with its dynamic views templates.
-                                $pool = new RollingCurl(array($this, 'handleCurlResponse'));
+                                                                // Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
-                                $pool->window_size = count($subset);            
+                                                                if (isset($this->requests[$orig]['body'])) {
-                                
+                                                                        $redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
-                                foreach ($subset as $orig => $url) {
+                                                                        if ($redirectURL) {
-                                        if (!$isRedirect) $orig = $url;
+                                                                                $this->redirectQueue[$orig] = $redirectURL;
-                                        unset($this->redirectQueue[$orig]);
+                                                                        }
-                                        $this->debug("...$url");
+                                                                }
-                                        if (!$isRedirect && isset($this->requests[$url])) {
+                                                        }
-                                                $this->debug("......in memory");
+                                                        //die($url.' -multi- '.$request->getResponseInfo('effective_url'));
-                                        /*
+                                                        $pool->detach($request);
-                                        } elseif ($this->isCached($url)) {
+                                                        unset($this->requests[$orig]['httpRequest'], $request);
-                                                $this->debug("......is cached");
+                                                        /*
-                                                if (!$this->minimiseMemoryUse) {
+                                                        if ($this->minimiseMemoryUse) {
-                                                        $this->requests[$url] = $this->getCached($url);
+                                                                if ($this->cache($url)) {
-                                                }
+                                                                        unset($this->requests[$url]);
-                                        */
+                                                                }
-                                        } else {
+                                                        }
-                                                $this->debug("......adding to pool");
+                                                        */
-                                                $req_url = $this->rewriteUrls($url);
+                                                }
-                                                $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;
+                                        }
-                                                $req_url = $this->removeFragment($req_url);
+                                }
-                                                if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) {
+                        } catch (HttpException $e) {
-                                                        $_meth = 'HEAD';
+                                $this->debug($e);
-                                                } else {
+                                return false;
-                                                        $_meth = 'GET';
+                        }
-                                                        unset($this->requests[$orig]['wrongGuess']);
+                }
-                                                }                                               
+                
-                                                $headers = array();
+                //////////////////////////////////////////////////////////
-                                                //$headers[] = 'User-Agent: '.$this->userAgent;
+                // parallel (curl_multi_*)
-                                                $headers[] = $this->getUserAgent($req_url);
+                elseif ($this->method == self::METHOD_CURL_MULTI) {
-                                                // add referer for picky sites
+                        $this->debug('Starting parallel fetch (curl_multi_*)');
-                                                $headers[] = 'Referer: '.$this->referer;
+                        while (count($urls) > 0) {
-                                                // send cookies, if we have any
+                                $this->debug('Processing set of '.min($this->maxParallelRequests, count($urls)));
-                                                if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
+                                $subset = array_splice($urls, 0, $this->maxParallelRequests);
-                                                        $this->debug("......sending cookies: $cookies");
+                                $pool = new RollingCurl(array($this, 'handleCurlResponse'));
-                                                        $headers[] = 'Cookie: '.$cookies;
+                                $pool->window_size = count($subset);            
-                                                }
+                                
-                                                $httpRequest = new RollingCurlRequest($req_url, $_meth, null, $headers, array(
+                                foreach ($subset as $orig => $url) {
-                                                        CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'],
+                                        if (!$isRedirect) $orig = $url;
-                                                        CURLOPT_TIMEOUT => $this->requestOptions['timeout']
+                                        unset($this->redirectQueue[$orig]);
-                                                        ));
+                                        $this->debug("...$url");
-                                                $httpRequest->set_original_url($orig);
+                                        if (!$isRedirect && isset($this->requests[$url])) {
-                                                $this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest);
+                                                $this->debug("......in memory");
-                                                $this->requests[$orig]['original_url'] = $orig; // TODO: is this needed anymore?
+                                        /*
-                                                $pool->add($httpRequest);
+                                        } elseif ($this->isCached($url)) {
-                                        }
+                                                $this->debug("......is cached");
-                                }
+                                                if (!$this->minimiseMemoryUse) {
-                                // did we get anything into the pool?
+                                                        $this->requests[$url] = $this->getCached($url);
-                                if (count($pool) > 0) {
+                                                }
-                                        $this->debug('Sending request...');
+                                        */
-                                        $pool->execute(); // this will call handleCurlResponse() and populate $this->requests[$orig]
+                                        } else {
-                                        $this->debug('Received responses');
+                                                $this->debug("......adding to pool");
-                                        foreach($subset as $orig => $url) {
+                                                $req_url = $this->rewriteUrls($url);
-                                                if (!$isRedirect) $orig = $url;
+                                                $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;
-                                                // $this->requests[$orig]['headers']
+                                                $req_url = $this->removeFragment($req_url);
-                                                // $this->requests[$orig]['body']
+                                                if (!empty($this->headerOnlyTypes) && !isset($this->requests[$orig]['wrongGuess']) && $this->possibleUnsupportedType($req_url)) {
-                                                // $this->requests[$orig]['effective_url']
+                                                        $_meth = 'HEAD';
-                                                // check content type
+                                                } else {
-                                                if ($this->headerOnlyType($this->requests[$orig]['headers'])) {
+                                                        $_meth = 'GET';
-                                                        $this->requests[$orig]['body'] = '';
+                                                        unset($this->requests[$orig]['wrongGuess']);
-                                                        $_header_only_type = true;
+                                                }                                               
-                                                        $this->debug('Header only type returned');
+                                                $headers = array();
-                                                } else {
+                                                //$headers[] = 'User-Agent: '.$this->userAgent;
-                                                        $_header_only_type = false;
+                                                $headers[] = $this->getUserAgent($req_url);
-                                                }
+                                                // add referer for picky sites
-                                                $status_code = $this->requests[$orig]['status_code'];
+                                                $headers[] = 'Referer: '.$this->referer;
-                                                if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
+                                                // send cookies, if we have any
-                                                        $redirectURL = $this->requests[$orig]['location'];
+                                                if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
-                                                        if (!preg_match('!^https?://!i', $redirectURL)) {
+                                                        $this->debug("......sending cookies: $cookies");
-                                                                $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
+                                                        $headers[] = 'Cookie: '.$cookies;
-                                                        }
+                                                }
-                                                        if ($this->validateURL($redirectURL)) {
+                                                $httpRequest = new RollingCurlRequest($req_url, $_meth, null, $headers, array(
-                                                                $this->debug('Redirect detected. Valid URL: '.$redirectURL);
+                                                        CURLOPT_CONNECTTIMEOUT => $this->requestOptions['timeout'],
-                                                                // store any cookies
+                                                        CURLOPT_TIMEOUT => $this->requestOptions['timeout']
-                                                                $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
+                                                        ));
-                                                                if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies);                                                   
+                                                $httpRequest->set_original_url($orig);
-                                                                $this->redirectQueue[$orig] = $redirectURL;
+                                                $this->requests[$orig] = array('headers'=>null, 'body'=>null, 'httpRequest'=>$httpRequest);
-                                                        } else {
+                                                $this->requests[$orig]['original_url'] = $orig; // TODO: is this needed anymore?
-                                                                $this->debug('Redirect detected. Invalid URL: '.$redirectURL);
+                                                $pool->add($httpRequest);
-                                                        }
+                                        }
-                                                } elseif (!$_header_only_type && $this->requests[$orig]['method'] == 'HEAD') {
+                                }
-                                                        // the response content-type did not match our 'header only' types, 
+                                // did we get anything into the pool?
-                                                        // but we'd issues a HEAD request because we assumed it would. So
+                                if (count($pool) > 0) {
-                                                        // let's queue a proper GET request for this item...
+                                        $this->debug('Sending request...');
-                                                        $this->debug('Wrong guess at content-type, queing GET request');
+                                        $pool->execute(); // this will call handleCurlResponse() and populate $this->requests[$orig]
-                                                        $this->requests[$orig]['wrongGuess'] = true;
+                                        $this->debug('Received responses');
-                                                        $this->redirectQueue[$orig] = $this->requests[$orig]['effective_url'];
+                                        foreach($subset as $orig => $url) {
-                                                } elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {
+                                                if (!$isRedirect) $orig = $url;
-                                                        // check for <meta name='fragment' content='!'/>
+                                                // $this->requests[$orig]['headers']
-                                                        // for AJAX sites, e.g. Blogger with its dynamic views templates.
+                                                // $this->requests[$orig]['body']
-                                                        // Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
+                                                // $this->requests[$orig]['effective_url']
-                                                        if (isset($this->requests[$orig]['body'])) {
+                                                // check content type
-                                                                $redirectURL = $this->getUglyURL($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
+                                                if ($this->headerOnlyType($this->requests[$orig]['headers'])) {
-                                                                if ($redirectURL) {
+                                                        $this->requests[$orig]['body'] = '';
-                                                                        $this->debug('AJAX trigger (meta name="fragment" content="!") found. Queueing '.$redirectURL);
+                                                        $_header_only_type = true;
-                                                                        $this->redirectQueue[$orig] = $redirectURL;
+                                                        $this->debug('Header only type returned');
-                                                                }
+                                                } else {
-                                                        }
+                                                        $_header_only_type = false;
-                                                }
+                                                }
-                                                // die($url.' -multi- '.$request->getResponseInfo('effective_url'));
+                                                $status_code = $this->requests[$orig]['status_code'];
-                                                unset($this->requests[$orig]['httpRequest'], $this->requests[$orig]['method']);
+                                                if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
-                                        }
+                                                        $redirectURL = $this->requests[$orig]['location'];
-                                }
+                                                        if (!preg_match('!^https?://!i', $redirectURL)) {
-                        }
+                                                                $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
-                }
+                                                        }
+                                                        if ($this->validateURL($redirectURL)) {
-                //////////////////////////////////////////////////////
+                                                                $this->debug('Redirect detected. Valid URL: '.$redirectURL);
-                // sequential (file_get_contents)
+                                                                // store any cookies
-                else {
+                                                                $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
-                        $this->debug('Starting sequential fetch (file_get_contents)');
+                                                                if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies);                                                   
-                        $this->debug('Processing set of '.count($urls));
+                                                                $this->redirectQueue[$orig] = $redirectURL;
-                        foreach ($urls as $orig => $url) {
+                                                        } else {
-                                if (!$isRedirect) $orig = $url;
+                                                                $this->debug('Redirect detected. Invalid URL: '.$redirectURL);
-                                unset($this->redirectQueue[$orig]);
+                                                        }
-                                $this->debug("...$url");
+                                                } elseif (!$_header_only_type && $this->requests[$orig]['method'] == 'HEAD') {
-                                if (!$isRedirect && isset($this->requests[$url])) {
+                                                        // the response content-type did not match our 'header only' types, 
-                                        $this->debug("......in memory");
+                                                        // but we'd issues a HEAD request because we assumed it would. So
-                                /*
+                                                        // let's queue a proper GET request for this item...
-                                } elseif ($this->isCached($url)) {
+                                                        $this->debug('Wrong guess at content-type, queing GET request');
-                                        $this->debug("......is cached");
+                                                        $this->requests[$orig]['wrongGuess'] = true;
-                                        if (!$this->minimiseMemoryUse) {
+                                                        $this->redirectQueue[$orig] = $this->requests[$orig]['effective_url'];
-                                                $this->requests[$url] = $this->getCached($url);
+                                                } elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {
-                                        }
+                                                        // check for <meta name='fragment' content='!'/>
-                                */
+                                                        // for AJAX sites, e.g. Blogger with its dynamic views templates.
-                                } else {
+                                                        // Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
-                                        $this->debug("Sending request for $url");
+                                                        if (isset($this->requests[$orig]['body'])) {
-                                        $this->requests[$orig]['original_url'] = $orig;
+                                                                $redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
-                                        $req_url = $this->rewriteUrls($url);
+                                                                if ($redirectURL) {
-                                        $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;
+                                                                        $this->redirectQueue[$orig] = $redirectURL;
-                                        $req_url = $this->removeFragment($req_url);
+                                                                }
-                                        // send cookies, if we have any
+                                                        }
-                                        $httpContext = $this->httpContext;
+                                                }
-                                        $httpContext['http']['header'] .= $this->getUserAgent($req_url)."\r\n";
+                                                // die($url.' -multi- '.$request->getResponseInfo('effective_url'));
-                                        // add referer for picky sites
+                                                unset($this->requests[$orig]['httpRequest'], $this->requests[$orig]['method']);
-                                        $httpContext['http']['header'] .= 'Referer: '.$this->referer."\r\n";
+                                        }
-                                        if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
+                                }
-                                                $this->debug("......sending cookies: $cookies");
+                        }
-                                                $httpContext['http']['header'] .= 'Cookie: '.$cookies."\r\n";
+                }
-                                        }
-                                        if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) {
+                //////////////////////////////////////////////////////
-                                                $this->debug('Received response');
+                // sequential (file_get_contents)
-                                                // get status code
+                else {
-                                                if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\d+\.\d+\s+(\d+)!', trim($http_response_header[0]), $match)) {
+                        $this->debug('Starting sequential fetch (file_get_contents)');
-                                                        $this->debug('Error: no status code found');
+                        $this->debug('Processing set of '.count($urls));
-                                                        // TODO: handle error - no status code
+                        foreach ($urls as $orig => $url) {
-                                                } else {
+                                if (!$isRedirect) $orig = $url;
-                                                        $this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false);
+                                unset($this->redirectQueue[$orig]);
-                                                        // check content type
+                                $this->debug("...$url");
-                                                        if ($this->headerOnlyType($this->requests[$orig]['headers'])) {
+                                if (!$isRedirect && isset($this->requests[$url])) {
-                                                                $this->requests[$orig]['body'] = '';
+                                        $this->debug("......in memory");
-                                                        } else {
+                                /*
-                                                                $this->requests[$orig]['body'] = $html;
+                                } elseif ($this->isCached($url)) {
-                                                        }
+                                        $this->debug("......is cached");
-                                                        $this->requests[$orig]['effective_url'] = $req_url;
+                                        if (!$this->minimiseMemoryUse) {
-                                                        $this->requests[$orig]['status_code'] = $status_code = (int)$match[1];
+                                                $this->requests[$url] = $this->getCached($url);
-                                                        unset($match);
+                                        }
-                                                        // handle redirect
+                                */
-                                                        if (preg_match('/^Location:(.*?)$/mi', $this->requests[$orig]['headers'], $match)) {
+                                } else {
-                                                                $this->requests[$orig]['location'] =  trim($match[1]);
+                                        $this->debug("Sending request for $url");
-                                                        }
+                                        $this->requests[$orig]['original_url'] = $orig;
-                                                        if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
+                                        $req_url = $this->rewriteUrls($url);
-                                                                $redirectURL = $this->requests[$orig]['location'];
+                                        $req_url = ($this->rewriteHashbangFragment) ? $this->rewriteHashbangFragment($req_url) : $req_url;
-                                                                if (!preg_match('!^https?://!i', $redirectURL)) {
+                                        $req_url = $this->removeFragment($req_url);
-                                                                        $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
+                                        // send cookies, if we have any
-                                                                }
+                                        $httpContext = $this->httpContext;
-                                                                if ($this->validateURL($redirectURL)) {
+                                        $httpContext['http']['header'] .= $this->getUserAgent($req_url)."\r\n";
-                                                                        $this->debug('Redirect detected. Valid URL: '.$redirectURL);
+                                        // add referer for picky sites
-                                                                        // store any cookies
+                                        $httpContext['http']['header'] .= 'Referer: '.$this->referer."\r\n";
-                                                                        $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
+                                        if ($cookies = $this->cookieJar->getMatchingCookies($req_url)) {
-                                                                        if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies);
+                                                $this->debug("......sending cookies: $cookies");
-                                                                        $this->redirectQueue[$orig] = $redirectURL;
+                                                $httpContext['http']['header'] .= 'Cookie: '.$cookies."\r\n";
-                                                                } else {
+                                        }
-                                                                        $this->debug('Redirect detected. Invalid URL: '.$redirectURL);
+                                        if (false !== ($html = @file_get_contents($req_url, false, stream_context_create($httpContext)))) {
-                                                                }
+                                                $this->debug('Received response');
-                                                        } elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {
+                                                // get status code
-                                                                // check for <meta name='fragment' content='!'/>
+                                                if (!isset($http_response_header[0]) || !preg_match('!^HTTP/\d+\.\d+\s+(\d+)!', trim($http_response_header[0]), $match)) {
-                                                                // for AJAX sites, e.g. Blogger with its dynamic views templates.
+                                                        $this->debug('Error: no status code found');
-                                                                // Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
+                                                        // TODO: handle error - no status code
-                                                                if (isset($this->requests[$orig]['body'])) {
+                                                } else {
-                                                                        $redirectURL = $this->getUglyURL($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
+                                                        $this->requests[$orig]['headers'] = $this->headersToString($http_response_header, false);
-                                                                        if ($redirectURL) {
+                                                        // check content type
-                                                                                $this->debug('AJAX trigger (meta name="fragment" content="!") found. Queueing '.$redirectURL);
+                                                        if ($this->headerOnlyType($this->requests[$orig]['headers'])) {
-                                                                                $this->redirectQueue[$orig] = $redirectURL;
+                                                                $this->requests[$orig]['body'] = '';
-                                                                        }
+                                                        } else {
-                                                                }
+                                                                $this->requests[$orig]['body'] = $html;
-                                                        }
+                                                        }
-                                                }
+                                                        $this->requests[$orig]['effective_url'] = $req_url;
-                                        } else {
+                                                        $this->requests[$orig]['status_code'] = $status_code = (int)$match[1];
-                                                $this->debug('Error retrieving URL');
+                                                        unset($match);
-                                                //print_r($req_url);
+                                                        // handle redirect
-                                                //print_r($http_response_header);
+                                                        if (preg_match('/^Location:(.*?)$/mi', $this->requests[$orig]['headers'], $match)) {
-                                                //print_r($html);
+                                                                $this->requests[$orig]['location'] =  trim($match[1]);
-                                                
+                                                        }
-                                                // TODO: handle error - failed to retrieve URL
+                                                        if ((in_array($status_code, array(300, 301, 302, 303, 307)) || $status_code > 307 && $status_code < 400) && isset($this->requests[$orig]['location'])) {
-                                        }
+                                                                $redirectURL = $this->requests[$orig]['location'];
-                                }
+                                                                if (!preg_match('!^https?://!i', $redirectURL)) {
-                        }
+                                                                        $redirectURL = SimplePie_Misc::absolutize_url($redirectURL, $url);
-                }
+                                                                }
-        }
+                                                                if ($this->validateURL($redirectURL)) {
-        
+                                                                        $this->debug('Redirect detected. Valid URL: '.$redirectURL);
-        public function handleCurlResponse($response, $info, $request) {
+                                                                        // store any cookies
-                $orig = $request->url_original;
+                                                                        $cookies = $this->cookieJar->extractCookies($this->requests[$orig]['headers']);
-                $this->requests[$orig]['headers'] = substr($response, 0, $info['header_size']);
+                                                                        if (!empty($cookies)) $this->cookieJar->storeCookies($url, $cookies);
-                $this->requests[$orig]['body'] = substr($response, $info['header_size']);
+                                                                        $this->redirectQueue[$orig] = $redirectURL;
-                $this->requests[$orig]['method'] = $request->method;
+                                                                } else {
-                $this->requests[$orig]['effective_url'] = $info['url'];
+                                                                        $this->debug('Redirect detected. Invalid URL: '.$redirectURL);
-                $this->requests[$orig]['status_code'] = (int)$info['http_code'];
+                                                                }
-                if (preg_match('/^Location:(.*?)$/mi', $this->requests[$orig]['headers'], $match)) {
+                                                        } elseif (strpos($this->requests[$orig]['effective_url'], '_escaped_fragment_') === false) {
-                        $this->requests[$orig]['location'] =  trim($match[1]);
+                                                                // check for <meta name='fragment' content='!'/>
-                }
+                                                                // for AJAX sites, e.g. Blogger with its dynamic views templates.
-        }
+                                                                // Based on Google's spec: https://developers.google.com/webmasters/ajax-crawling/docs/specification
-        
+                                                                if (isset($this->requests[$orig]['body'])) {
-        protected function headersToString(array $headers, $associative=true) {
+                                                                        $redirectURL = $this->getRedirectURLfromHTML($this->requests[$orig]['effective_url'], substr($this->requests[$orig]['body'], 0, 4000));
-                if (!$associative) {
+                                                                        if ($redirectURL) {
-                        return implode("\n", $headers);
+                                                                                $this->redirectQueue[$orig] = $redirectURL;
-                } else {
+                                                                        }
-                        $str = '';
+                                                                }
-                        foreach ($headers as $key => $val) {
+                                                        }
-                                if (is_array($val)) {
+                                                }
-                                        foreach ($val as $v) $str .= "$key: $v\n";
+                                        } else {
-                                } else {
+                                                $this->debug('Error retrieving URL');
-                                        $str .= "$key: $val\n";
+                                                //print_r($req_url);
-                                }
+                                                //print_r($http_response_header);
-                        }
+                                                //print_r($html);
-                        return rtrim($str);
+                                                
-                }
+                                                // TODO: handle error - failed to retrieve URL
-        }
+                                        }
-        
+                                }
-        public function get($url, $remove=false, $gzdecode=true) {
+                        }
-                $url = "$url";
+                }
-                if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) {
+        }
-                        $this->debug("URL already fetched - in memory ($url, effective: {$this->requests[$url]['effective_url']})");
+        
-                        $response = $this->requests[$url];
+        public function handleCurlResponse($response, $info, $request) {
-                /*
+                $orig = $request->url_original;
-                } elseif ($this->isCached($url)) {
+                $this->requests[$orig]['headers'] = substr($response, 0, $info['header_size']);
-                        $this->debug("URL already fetched - in disk cache ($url)");
+                $this->requests[$orig]['body'] = substr($response, $info['header_size']);
-                        $response = $this->getCached($url);
+                $this->requests[$orig]['method'] = $request->method;
-                        $this->requests[$url] = $response;
+                $this->requests[$orig]['effective_url'] = $info['url'];
-                */
+                $this->requests[$orig]['status_code'] = (int)$info['http_code'];
-                } else {
+                if (preg_match('/^Location:(.*?)$/mi', $this->requests[$orig]['headers'], $match)) {
-                        $this->debug("Fetching URL ($url)");
+                        $this->requests[$orig]['location'] =  trim($match[1]);
-                        $this->fetchAll(array($url));
+                }
-                        if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) {
+        }
-                                $response = $this->requests[$url];
+        
-                        } else {
+        protected function headersToString(array $headers, $associative=true) {
-                                $this->debug("Request failed");
+                if (!$associative) {
-                                $response = false;
+                        return implode("\n", $headers);
-                        }
+                } else {
-                }
+                        $str = '';
-                /*
+                        foreach ($headers as $key => $val) {
-                if ($this->minimiseMemoryUse && $response) {
+                                if (is_array($val)) {
-                        $this->cache($url);
+                                        foreach ($val as $v) $str .= "$key: $v\n";
-                        unset($this->requests[$url]);
+                                } else {
-                }
+                                        $str .= "$key: $val\n";
-                */
+                                }
-                if ($remove && $response) unset($this->requests[$url]);
+                        }
-                if ($gzdecode && stripos($response['headers'], 'Content-Encoding: gzip')) {
+                        return rtrim($str);
-                        if ($html = gzdecode($response['body'])) {
+                }
-                                $response['body'] = $html;
+        }
-                        }
+        
-                }
+        public function get($url, $remove=false, $gzdecode=true) {
-                return $response;
+                $url = "$url";
-        }
+                if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) {
-        
+                        $this->debug("URL already fetched - in memory ($url, effective: {$this->requests[$url]['effective_url']})");
-        public function parallelSupport() {
+                        $response = $this->requests[$url];
-                return class_exists('HttpRequestPool') || function_exists('curl_multi_init');
+                /*
-        }
+                } elseif ($this->isCached($url)) {
-        
+                        $this->debug("URL already fetched - in disk cache ($url)");
-        private function headerOnlyType($headers) {
+                        $response = $this->getCached($url);
-                if (preg_match('!^Content-Type:\s*(([a-z-]+)/([^;\r\n ]+))!im', $headers, $match)) {
+                        $this->requests[$url] = $response;
-                        // look for full mime type (e.g. image/jpeg) or just type (e.g. image)
+                */
-                        $match[1] = strtolower(trim($match[1]));
+                } else {
-                        $match[2] = strtolower(trim($match[2]));
+                        $this->debug("Fetching URL ($url)");
-                        foreach (array($match[1], $match[2]) as $mime) {
+                        $this->fetchAll(array($url));
-                                if (in_array($mime, $this->headerOnlyTypes)) return true;
+                        if (isset($this->requests[$url]) && isset($this->requests[$url]['body'])) {
-                        }
+                                $response = $this->requests[$url];
-                }
+                        } else {
-                return false;
+                                $this->debug("Request failed");
-        }
+                                $response = false;
-        
+                        }
-        private function possibleUnsupportedType($url) {
+                }
-                $path = @parse_url($url, PHP_URL_PATH);
+                /*
-                if ($path && strpos($path, '.') !== false) {
+                if ($this->minimiseMemoryUse && $response) {
-                        $ext = strtolower(trim(pathinfo($path, PATHINFO_EXTENSION)));
+                        $this->cache($url);
-                        return in_array($ext, $this->headerOnlyClues);
+                        unset($this->requests[$url]);
-                }
+                }
-                return false;
+                */
-        }
+                if ($remove && $response) unset($this->requests[$url]);
-}
+                if ($gzdecode && stripos($response['headers'], 'Content-Encoding: gzip')) {
+                        if ($html = gzdecode($response['body'])) {
-// gzdecode from http://www.php.net/manual/en/function.gzdecode.php#82930
+                                $response['body'] = $html;
-if (!function_exists('gzdecode')) {
+                        }
-        function gzdecode($data,&$filename='',&$error='',$maxlength=null) 
+                }
-        {
+                return $response;
-                $len = strlen($data);
+        }
-                if ($len < 18 || strcmp(substr($data,0,2),"\x1f\x8b")) {
+        
-                        $error = "Not in GZIP format.";
+        public function parallelSupport() {
-                        return null;  // Not GZIP format (See RFC 1952)
+                return class_exists('HttpRequestPool') || function_exists('curl_multi_init');
-                }
+        }
-                $method = ord(substr($data,2,1));  // Compression method
+        
-                $flags  = ord(substr($data,3,1));  // Flags
+        private function headerOnlyType($headers) {
-                if ($flags & 31 != $flags) {
+                if (preg_match('!^Content-Type:\s*(([a-z-]+)/([^;\r\n ]+))!im', $headers, $match)) {
-                        $error = "Reserved bits not allowed.";
+                        // look for full mime type (e.g. image/jpeg) or just type (e.g. image)
-                        return null;
+                        $match[1] = strtolower(trim($match[1]));
-                }
+                        $match[2] = strtolower(trim($match[2]));
-                // NOTE: $mtime may be negative (PHP integer limitations)
+                        foreach (array($match[1], $match[2]) as $mime) {
-                $mtime = unpack("V", substr($data,4,4));
+                                if (in_array($mime, $this->headerOnlyTypes)) return true;
-                $mtime = $mtime[1];
+                        }
-                $xfl   = substr($data,8,1);
+                }
-                $os    = substr($data,8,1);
+                return false;
-                $headerlen = 10;
+        }
-                $extralen  = 0;
+        
-                $extra     = "";
+        private function possibleUnsupportedType($url) {
-                if ($flags & 4) {
+                $path = @parse_url($url, PHP_URL_PATH);
-                        // 2-byte length prefixed EXTRA data in header
+                if ($path && strpos($path, '.') !== false) {
-                        if ($len - $headerlen - 2 < 8) {
+                        $ext = strtolower(trim(pathinfo($path, PATHINFO_EXTENSION)));
-                                return false;  // invalid
+                        return in_array($ext, $this->headerOnlyClues);
-                        }
+                }
-                        $extralen = unpack("v",substr($data,8,2));
+                return false;
-                        $extralen = $extralen[1];
+        }
-                        if ($len - $headerlen - 2 - $extralen < 8) {
+}
-                                return false;  // invalid
-                        }
+// gzdecode from http://www.php.net/manual/en/function.gzdecode.php#82930
-                        $extra = substr($data,10,$extralen);
+if (!function_exists('gzdecode')) {
-                        $headerlen += 2 + $extralen;
+        function gzdecode($data,&$filename='',&$error='',$maxlength=null) 
-                }
+        {
-                $filenamelen = 0;
+                $len = strlen($data);
-                $filename = "";
+                if ($len < 18 || strcmp(substr($data,0,2),"\x1f\x8b")) {
-                if ($flags & 8) {
+                        $error = "Not in GZIP format.";
-                        // C-style string
+                        return null;  // Not GZIP format (See RFC 1952)
-                        if ($len - $headerlen - 1 < 8) {
+                }
-                                return false; // invalid
+                $method = ord(substr($data,2,1));  // Compression method
-                        }
+                $flags  = ord(substr($data,3,1));  // Flags
-                        $filenamelen = strpos(substr($data,$headerlen),chr(0));
+                if ($flags & 31 != $flags) {
-                        if ($filenamelen === false || $len - $headerlen - $filenamelen - 1 < 8) {
+                        $error = "Reserved bits not allowed.";
-                                return false; // invalid
+                        return null;
-                        }
+                }
-                        $filename = substr($data,$headerlen,$filenamelen);
+                // NOTE: $mtime may be negative (PHP integer limitations)
-                        $headerlen += $filenamelen + 1;
+                $mtime = unpack("V", substr($data,4,4));
-                }
+                $mtime = $mtime[1];
-                $commentlen = 0;
+                $xfl   = substr($data,8,1);
-                $comment = "";
+                $os    = substr($data,8,1);
-                if ($flags & 16) {
+                $headerlen = 10;
-                        // C-style string COMMENT data in header
+                $extralen  = 0;
-                        if ($len - $headerlen - 1 < 8) {
+                $extra     = "";
-                                return false;    // invalid
+                if ($flags & 4) {
-                        }
+                        // 2-byte length prefixed EXTRA data in header
-                        $commentlen = strpos(substr($data,$headerlen),chr(0));
+                        if ($len - $headerlen - 2 < 8) {
-                        if ($commentlen === false || $len - $headerlen - $commentlen - 1 < 8) {
+                                return false;  // invalid
-                                return false;    // Invalid header format
+                        }
-                        }
+                        $extralen = unpack("v",substr($data,8,2));
-                        $comment = substr($data,$headerlen,$commentlen);
+                        $extralen = $extralen[1];
-                        $headerlen += $commentlen + 1;
+                        if ($len - $headerlen - 2 - $extralen < 8) {
-                }
+                                return false;  // invalid
-                $headercrc = "";
+                        }
-                if ($flags & 2) {
+                        $extra = substr($data,10,$extralen);
-                        // 2-bytes (lowest order) of CRC32 on header present
+                        $headerlen += 2 + $extralen;
-                        if ($len - $headerlen - 2 < 8) {
+                }
-                                return false;    // invalid
+                $filenamelen = 0;
-                        }
+                $filename = "";
-                        $calccrc = crc32(substr($data,0,$headerlen)) & 0xffff;
+                if ($flags & 8) {
-                        $headercrc = unpack("v", substr($data,$headerlen,2));
+                        // C-style string
-                        $headercrc = $headercrc[1];
+                        if ($len - $headerlen - 1 < 8) {
-                        if ($headercrc != $calccrc) {
+                                return false; // invalid
-                                $error = "Header checksum failed.";
+                        }
-                                return false;    // Bad header CRC
+                        $filenamelen = strpos(substr($data,$headerlen),chr(0));
-                        }
+                        if ($filenamelen === false || $len - $headerlen - $filenamelen - 1 < 8) {
-                        $headerlen += 2;
+                                return false; // invalid
-                }
+                        }
-                // GZIP FOOTER
+                        $filename = substr($data,$headerlen,$filenamelen);
-                $datacrc = unpack("V",substr($data,-8,4));
+                        $headerlen += $filenamelen + 1;
-                $datacrc = sprintf('%u',$datacrc[1] & 0xFFFFFFFF);
+                }
-                $isize = unpack("V",substr($data,-4));
+                $commentlen = 0;
-                $isize = $isize[1];
+                $comment = "";
-                // decompression:
+                if ($flags & 16) {
-                $bodylen = $len-$headerlen-8;
+                        // C-style string COMMENT data in header
-                if ($bodylen < 1) {
+                        if ($len - $headerlen - 1 < 8) {
-                        // IMPLEMENTATION BUG!
+                                return false;    // invalid
-                        return null;
+                        }
-                }
+                        $commentlen = strpos(substr($data,$headerlen),chr(0));
-                $body = substr($data,$headerlen,$bodylen);
+                        if ($commentlen === false || $len - $headerlen - $commentlen - 1 < 8) {
-                $data = "";
+                                return false;    // Invalid header format
-                if ($bodylen > 0) {
+                        }
-                        switch ($method) {
+                        $comment = substr($data,$headerlen,$commentlen);
-                        case 8:
+                        $headerlen += $commentlen + 1;
-                                // Currently the only supported compression method:
+                }
-                                $data = gzinflate($body,$maxlength);
+                $headercrc = "";
-                                break;
+                if ($flags & 2) {
-                        default:
+                        // 2-bytes (lowest order) of CRC32 on header present
-                                $error = "Unknown compression method.";
+                        if ($len - $headerlen - 2 < 8) {
-                                return false;
+                                return false;    // invalid
-                        }
+                        }
-                }  // zero-byte body content is allowed
+                        $calccrc = crc32(substr($data,0,$headerlen)) & 0xffff;
-                // Verifiy CRC32
+                        $headercrc = unpack("v", substr($data,$headerlen,2));
-                $crc   = sprintf("%u",crc32($data));
+                        $headercrc = $headercrc[1];
-                $crcOK = $crc == $datacrc;
+                        if ($headercrc != $calccrc) {
-                $lenOK = $isize == strlen($data);
+                                $error = "Header checksum failed.";
-                if (!$lenOK || !$crcOK) {
+                                return false;    // Bad header CRC
-                        $error = ( $lenOK ? '' : 'Length check FAILED. ') . ( $crcOK ? '' : 'Checksum FAILED.');
+                        }
-                        return false;
+                        $headerlen += 2;
-                }
+                }
-                return $data;
+                // GZIP FOOTER
-        }
+                $datacrc = unpack("V",substr($data,-8,4));
-}
+                $datacrc = sprintf('%u',$datacrc[1] & 0xFFFFFFFF);
-?>
-\ No newline at end of file
+                $isize = unpack("V",substr($data,-4));
+                $isize = $isize[1];
+                // decompression:
+                $bodylen = $len-$headerlen-8;
+                if ($bodylen < 1) {
+                        // IMPLEMENTATION BUG!
+                        return null;
+                }
+                $body = substr($data,$headerlen,$bodylen);
+                $data = "";
+                if ($bodylen > 0) {
+                        switch ($method) {
+                        case 8:
+                                // Currently the only supported compression method:
+                                $data = gzinflate($body,$maxlength);
+                                break;
+                        default:
+                                $error = "Unknown compression method.";
+                                return false;
+                        }
+                }  // zero-byte body content is allowed
+                // Verifiy CRC32
+                $crc   = sprintf("%u",crc32($data));
+                $crcOK = $crc == $datacrc;
+                $lenOK = $isize == strlen($data);
+                if (!$lenOK || !$crcOK) {
+                        $error = ( $lenOK ? '' : 'Length check FAILED. ') . ( $crcOK ? '' : 'Checksum FAILED.');
+                        return false;
+                }
+                return $data;
+        }
+}
+\ No newline at end of file
diff --git a/inc/3rdparty/libraries/humble-http-agent/SimplePie_HumbleHttpAgent.php b/inc/3rdparty/libraries/humble-http-agent/SimplePie_HumbleHttpAgent.php
index ecd46d5f..c524a1ee 100644
--- a/inc/3rdparty/libraries/humble-http-agent/SimplePie_HumbleHttpAgent.php
+++ b/inc/3rdparty/libraries/humble-http-agent/SimplePie_HumbleHttpAgent.php
@@ -1,79 +1,78 @@
-<?php
+<?php
-/**
+/**
- * Humble HTTP Agent extension for SimplePie_File
+ * Humble HTTP Agent extension for SimplePie_File
- * 
+ * 
- * This class is designed to extend and override SimplePie_File
+ * This class is designed to extend and override SimplePie_File
- * in order to prevent duplicate HTTP requests being sent out.
+ * in order to prevent duplicate HTTP requests being sent out.
- * The idea is to initialise an instance of Humble HTTP Agent
+ * The idea is to initialise an instance of Humble HTTP Agent
- * and attach it, to a static class variable, of this class.
+ * and attach it, to a static class variable, of this class.
- * SimplePie will then automatically initialise this class
+ * SimplePie will then automatically initialise this class
- * 
+ * 
- * @date 2011-02-28
+ * @date 2011-02-28
- */
+ */
-class SimplePie_HumbleHttpAgent extends SimplePie_File
+class SimplePie_HumbleHttpAgent extends SimplePie_File
-{
+{
-        protected static $agent;
+        protected static $agent;
-        var $url;
+        var $url;
-        var $useragent;
+        var $useragent;
-        var $success = true;
+        var $success = true;
-        var $headers = array();
+        var $headers = array();
-        var $body;
+        var $body;
-        var $status_code;
+        var $status_code;
-        var $redirects = 0;
+        var $redirects = 0;
-        var $error;
+        var $error;
-        var $method = SIMPLEPIE_FILE_SOURCE_NONE;
+        var $method = SIMPLEPIE_FILE_SOURCE_NONE;
-        public static function set_agent(HumbleHttpAgent $agent) {
+        public static function set_agent(HumbleHttpAgent $agent) {
-                self::$agent = $agent;
+                self::$agent = $agent;
-        }
+        }
-        
+        
-        public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false) {
+        public function __construct($url, $timeout = 10, $redirects = 5, $headers = null, $useragent = null, $force_fsockopen = false) {
-                if (class_exists('idna_convert'))
+                if (class_exists('idna_convert'))
-                {
+                {
-                        $idn = new idna_convert();
+                        $idn = new idna_convert();
-                        $parsed = SimplePie_Misc::parse_url($url);
+                        $parsed = SimplePie_Misc::parse_url($url);
-                        $url = SimplePie_Misc::compress_parse_url($parsed['scheme'], $idn->encode($parsed['authority']), $parsed['path'], $parsed['query'], $parsed['fragment']);
+                        $url = SimplePie_Misc::compress_parse_url($parsed['scheme'], $idn->encode($parsed['authority']), $parsed['path'], $parsed['query'], $parsed['fragment']);
-                }
+                }
-                $this->url = $url;
+                $this->url = $url;
-                $this->useragent = $useragent;
+                $this->useragent = $useragent;
-                if (preg_match('/^http(s)?:\/\//i', $url))
+                if (preg_match('/^http(s)?:\/\//i', $url))
-                {
+                {
-                        if (!is_array($headers))
+                        if (!is_array($headers))
-                        {
+                        {
-                                $headers = array();
+                                $headers = array();
-                        }
+                        }
-                        $this->method = SIMPLEPIE_FILE_SOURCE_REMOTE | SIMPLEPIE_FILE_SOURCE_CURL;
+                        $this->method = SIMPLEPIE_FILE_SOURCE_REMOTE | SIMPLEPIE_FILE_SOURCE_CURL;
-                        $headers2 = array();
+                        $headers2 = array();
-                        foreach ($headers as $key => $value) {
+                        foreach ($headers as $key => $value) {
-                                $headers2[] = "$key: $value";
+                                $headers2[] = "$key: $value";
-                        }
+                        }
-                        //TODO: allow for HTTP headers
+                        //TODO: allow for HTTP headers
-                        // curl_setopt($fp, CURLOPT_HTTPHEADER, $headers2);
+                        // curl_setopt($fp, CURLOPT_HTTPHEADER, $headers2);
-                        $response = self::$agent->get($url);
+                        $response = self::$agent->get($url);
-                        
+                        
-                        if ($response === false || !isset($response['status_code'])) {
+                        if ($response === false || !isset($response['status_code'])) {
-                                $this->error = 'failed to fetch URL';
+                                $this->error = 'failed to fetch URL';
-                                $this->success = false;
+                                $this->success = false;
-                        } else {
+                        } else {
-                                // The extra lines at the end are there to satisfy SimplePie's HTTP parser.
+                                // The extra lines at the end are there to satisfy SimplePie's HTTP parser.
-                                // The class expects a full HTTP message, whereas we're giving it only
+                                // The class expects a full HTTP message, whereas we're giving it only
-                                // headers - the new lines indicate the start of the body.
+                                // headers - the new lines indicate the start of the body.
-                                $parser = new SimplePie_HTTP_Parser($response['headers']."\r\n\r\n");
+                                $parser = new SimplePie_HTTP_Parser($response['headers']."\r\n\r\n");
-                                if ($parser->parse()) {
+                                if ($parser->parse()) {
-                                        $this->headers = $parser->headers;
+                                        $this->headers = $parser->headers;
-                                        //$this->body = $parser->body;
+                                        //$this->body = $parser->body;
-                                        $this->body = $response['body'];
+                                        $this->body = $response['body'];
-                                        $this->status_code = $parser->status_code;
+                                        $this->status_code = $parser->status_code;
-                                }
+                                }
-                        }
+                        }
-                }
+                }
-                else
+                else
-                {
+                {
-                        $this->error = 'invalid URL';
+                        $this->error = 'invalid URL';
-                        $this->success = false;
+                        $this->success = false;
-                }
+                }
-        }
+        }
-}
+}
+\ No newline at end of file
-?>
-\ No newline at end of file
author	Nicolas Lœuillet <nicolas@loeuillet.org>	2014-05-29 12:50:28 +0200
committer	Nicolas Lœuillet <nicolas@loeuillet.org>	2014-05-29 12:50:28 +0200
commit	87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b (patch)
tree	558818975ac41403e7d55ad07c5b0ac29806e907 /inc/3rdparty/libraries/humble-http-agent
parent	ab157bbb75ba226917145c9bf906cbf764a85cd0 (diff)
parent	0b9bb8cb7868f24137c5d8b85c39cc88ea877411 (diff)
download	wallabag-87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b.tar.gz wallabag-87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b.tar.zst wallabag-87f01ea2e97715ac5df4ef7a6741cc26f3a5cd1b.zip