update to 3.2 version of full-text-rss, issue #694

1 files changed, 1137 insertions, 1137 deletions

diff --git a/inc/3rdparty/libraries/readability/Readability.php b/inc/3rdparty/libraries/readability/Readability.php
index 2e8991cc..d0f09d74 100644
--- a/inc/3rdparty/libraries/readability/Readability.php
+++ b/inc/3rdparty/libraries/readability/Readability.php
@@ -1,1138 +1,1138 @@
-<?php

-/** 

-* Arc90's Readability ported to PHP for FiveFilters.org

-* Based on readability.js version 1.7.1 (without multi-page support)

-* Updated to allow HTML5 parsing with html5lib

-* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds

-* ------------------------------------------------------

-* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js

-* Arc90's project URL: http://lab.arc90.com/experiments/readability/

-* JS Source: http://code.google.com/p/arc90labs-readability

-* Ported by: Keyvan Minoukadeh, http://www.keyvan.net

-* More information: http://fivefilters.org/content-only/

-* License: Apache License, Version 2.0

-* Requires: PHP5

-* Date: 2012-09-19

-* 

-* Differences between the PHP port and the original

-* ------------------------------------------------------

-* Arc90's Readability is designed to run in the browser. It works on the DOM 

-* tree (the parsed HTML) after the page's CSS styles have been applied and 

-* Javascript code executed. This PHP port does not run inside a browser. 

-* We use PHP's ability to parse HTML to build our DOM tree, but we cannot 

-* rely on CSS or Javascript support. As such, the results will not always 

-* match Arc90's Readability. (For example, if a web page contains CSS style 

-* rules or Javascript code which hide certain HTML elements from display, 

-* Arc90's Readability will dismiss those from consideration but our PHP port, 

-* unable to understand CSS or Javascript, will not know any better.)

-* 

-* Another significant difference is that the aim of Arc90's Readability is 

-* to re-present the main content block of a given web page so users can 

-* read it more easily in their browsers. Correct identification, clean up, 

-* and separation of the content block is only a part of this process. 

-* This PHP port is only concerned with this part, it does not include code 

-* that relates to presentation in the browser - Arc90 already do 

-* that extremely well, and for PDF output there's FiveFilters.org's 

-* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.

-* 

-* Finally, this class contains methods that might be useful for developers 

-* working on HTML document fragments. So without deviating too much from 

-* the original code (which I don't want to do because it makes debugging 

-* and updating more difficult), I've tried to make it a little more 

-* developer friendly. You should be able to use the methods here on 

-* existing DOMElement objects without passing an entire HTML document to 

-* be parsed.

-*/

-

-// This class allows us to do JavaScript like assignements to innerHTML

-require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');

-

-// Alternative usage (for testing only!)

-// uncomment the lines below and call Readability.php in your browser 

-// passing it the URL of the page you'd like content from, e.g.:

-// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php

-

-/*

-if (!isset($_GET['url']) || $_GET['url'] == '') {

-        die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');

-}

-$url = $_GET['url'];

-if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;

-$html = file_get_contents($url);

-$r = new Readability($html, $url);

-$r->init();

-echo $r->articleContent->innerHTML;

-*/

-

-class Readability

-{

-        public $version = '1.7.1-without-multi-page';

-        public $convertLinksToFootnotes = false;

-        public $revertForcedParagraphElements = true;

-        public $articleTitle;

-        public $articleContent;

-        public $dom;

-        public $url = null; // optional - URL where HTML was retrieved

-        public $debug = false;

-        public $lightClean = true; // preserves more content (experimental) added 2012-09-19

-        protected $body = null; // 

-        protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later

-        protected $flags = 7; // 1 | 2 | 4;   // Start with all flags set.

-        protected $success = false; // indicates whether we were able to extract or not

-        

-        /**

-        * All of the regular expressions in use within readability.

-        * Defined up here so we don't instantiate them repeatedly in loops.

-        **/

-        public $regexps = array(

-                'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',

-                'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',

-                'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',

-                'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',

-                'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',

-                'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',

-                'replaceFonts' => '/<(\/?)font[^>]*>/i',

-                // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()

-                'normalize' => '/\s{2,}/',

-                'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',

-                'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',

-                'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'

-        );      

-        

-        /* constants */

-        const FLAG_STRIP_UNLIKELYS = 1;

-        const FLAG_WEIGHT_CLASSES = 2;

-        const FLAG_CLEAN_CONDITIONALLY = 4;

-        

-        /**

-        * Create instance of Readability

-        * @param string UTF-8 encoded string

-        * @param string (optional) URL associated with HTML (used for footnotes)

-        * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')

-        */      

-        function __construct($html, $url=null, $parser='libxml')

-        {

-                $this->url = $url;

-                /* Turn all double br's into p's */

-                $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);

-                $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);

-                $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");

-                if (trim($html) == '') $html = '<html></html>';

-                if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {

-                        // all good

-                } else {

-                        $this->dom = new DOMDocument();

-                        $this->dom->preserveWhiteSpace = false;

-                        @$this->dom->loadHTML($html);

-                }

-                $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');

-        }

-

-        /**

-        * Get article title element

-        * @return DOMElement

-        */

-        public function getTitle() {

-                return $this->articleTitle;

-        }

-        

-        /**

-        * Get article content element

-        * @return DOMElement

-        */

-        public function getContent() {

-                return $this->articleContent;

-        }       

-        

-        /**

-        * Runs readability.

-        * 

-        * Workflow:

-        *  1. Prep the document by removing script tags, css, etc.

-        *  2. Build readability's DOM tree.

-        *  3. Grab the article content from the current dom tree.

-        *  4. Replace the current DOM tree with the new one.

-        *  5. Read peacefully.

-        *

-        * @return boolean true if we found content, false otherwise

-        **/

-        public function init()

-        {

-                if (!isset($this->dom->documentElement)) return false;

-                $this->removeScripts($this->dom);

-                //die($this->getInnerHTML($this->dom->documentElement));

-                

-                // Assume successful outcome

-                $this->success = true;

-

-                $bodyElems = $this->dom->getElementsByTagName('body');

-                if ($bodyElems->length > 0) {

-                        if ($this->bodyCache == null) {

-                                $this->bodyCache = $bodyElems->item(0)->innerHTML;

-                        }

-                        if ($this->body == null) {

-                                $this->body = $bodyElems->item(0);

-                        }

-                }

-

-                $this->prepDocument();

-                

-                //die($this->dom->documentElement->parentNode->nodeType);

-                //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));

-                //die($this->getInnerHTML($this->dom->documentElement));

-

-                /* Build readability's DOM tree */

-                $overlay        = $this->dom->createElement('div');

-                $innerDiv       = $this->dom->createElement('div');

-                $articleTitle   = $this->getArticleTitle();

-                $articleContent = $this->grabArticle();

-

-                if (!$articleContent) {

-                        $this->success = false;

-                        $articleContent = $this->dom->createElement('div');

-                        $articleContent->setAttribute('id', 'readability-content');

-                        $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';            

-                }

-                

-                $overlay->setAttribute('id', 'readOverlay');

-                $innerDiv->setAttribute('id', 'readInner');

-

-                /* Glue the structure of our document together. */

-                $innerDiv->appendChild($articleTitle);

-                $innerDiv->appendChild($articleContent);

-                $overlay->appendChild($innerDiv);

-                

-                /* Clear the old HTML, insert the new content. */

-                $this->body->innerHTML = '';

-                $this->body->appendChild($overlay);

-                //document.body.insertBefore(overlay, document.body.firstChild);

-                $this->body->removeAttribute('style');

-

-                $this->postProcessContent($articleContent);

-                

-                // Set title and content instance variables

-                $this->articleTitle = $articleTitle;

-                $this->articleContent = $articleContent;

-                

-                return $this->success;

-        }

-        

-        /**

-        * Debug

-        */

-        protected function dbg($msg) {

-                if ($this->debug) echo '* ',$msg, "\n";

-        }

-        

-        /**

-        * Run any post-process modifications to article content as necessary.

-        *

-        * @param DOMElement

-        * @return void

-        */

-        public function postProcessContent($articleContent) {

-                if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { 

-                        $this->addFootnotes($articleContent);

-                }

-        }

-        

-        /**

-        * Get the article title as an H1.

-        *

-        * @return DOMElement

-        */

-        protected function getArticleTitle() {

-                $curTitle = '';

-                $origTitle = '';

-

-                try {

-                        $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));

-                } catch(Exception $e) {}

-                

-                if (preg_match('/ [\|\-] /', $curTitle))

-                {

-                        $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);

-                        

-                        if (count(explode(' ', $curTitle)) < 3) {

-                                $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);

-                        }

-                }

-                else if (strpos($curTitle, ': ') !== false)

-                {

-                        $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);

-

-                        if (count(explode(' ', $curTitle)) < 3) {

-                                $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);

-                        }

-                }

-                else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)

-                {

-                        $hOnes = $this->dom->getElementsByTagName('h1');

-                        if($hOnes->length == 1)

-                        {

-                                $curTitle = $this->getInnerText($hOnes->item(0));

-                        }

-                }

-

-                $curTitle = trim($curTitle);

-

-                if (count(explode(' ', $curTitle)) <= 4) {

-                        $curTitle = $origTitle;

-                }

-                

-                $articleTitle = $this->dom->createElement('h1');

-                $articleTitle->innerHTML = $curTitle;

-                

-                return $articleTitle;

-        }

-        

-        /**

-        * Prepare the HTML document for readability to scrape it.

-        * This includes things like stripping javascript, CSS, and handling terrible markup.

-        * 

-        * @return void

-        **/

-        protected function prepDocument() {

-                /**

-                * In some cases a body element can't be found (if the HTML is totally hosed for example)

-                * so we create a new body node and append it to the document.

-                */

-                if ($this->body == null)

-                {

-                        $this->body = $this->dom->createElement('body');

-                        $this->dom->documentElement->appendChild($this->body);

-                }

-                $this->body->setAttribute('id', 'readabilityBody');

-

-                /* Remove all style tags in head */

-                $styleTags = $this->dom->getElementsByTagName('style');

-                for ($i = $styleTags->length-1; $i >= 0; $i--)

-                {

-                        $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));

-                }

-

-                /* Turn all double br's into p's */

-                /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */

-                //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');

-                // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.

-                // Manipulating innerHTML as it's done in JS is not possible in PHP.

-        }

-

-        /**

-        * For easier reading, convert this document to have footnotes at the bottom rather than inline links.

-        * @see http://www.roughtype.com/archives/2010/05/experiments_in.php

-        *

-        * @return void

-        **/

-        public function addFootnotes($articleContent) {

-                $footnotesWrapper = $this->dom->createElement('div');

-                $footnotesWrapper->setAttribute('id', 'readability-footnotes');

-                $footnotesWrapper->innerHTML = '<h3>References</h3>';

-                

-                $articleFootnotes = $this->dom->createElement('ol');

-                $articleFootnotes->setAttribute('id', 'readability-footnotes-list');

-                $footnotesWrapper->appendChild($articleFootnotes);

-                

-                $articleLinks = $articleContent->getElementsByTagName('a');

-                

-                $linkCount = 0;

-                for ($i = 0; $i < $articleLinks->length; $i++)

-                {

-                        $articleLink  = $articleLinks->item($i);

-                        $footnoteLink = $articleLink->cloneNode(true);

-                        $refLink      = $this->dom->createElement('a');

-                        $footnote     = $this->dom->createElement('li');

-                        $linkDomain   = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);

-                        if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);

-                        //linkDomain   = footnoteLink.host ? footnoteLink.host : document.location.host,

-                        $linkText     = $this->getInnerText($articleLink);

-                        

-                        if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {

-                                continue;

-                        }

-                        

-                        $linkCount++;

-

-                        /** Add a superscript reference after the article link */

-                        $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);

-                        $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';

-                        $refLink->setAttribute('class', 'readability-DoNotFootnote');

-                        $refLink->setAttribute('style', 'color: inherit;');

-                        

-                        //TODO: does this work or should we use DOMNode.isSameNode()?

-                        if ($articleLink->parentNode->lastChild == $articleLink) {

-                                $articleLink->parentNode->appendChild($refLink);

-                        } else {

-                                $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);

-                        }

-

-                        $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');

-                        $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);

-

-                        $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';

-

-                        $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);

-                        $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);

-                        

-                        $footnote->appendChild($footnoteLink);

-                        if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';

-                        

-                        $articleFootnotes->appendChild($footnote);

-                }

-

-                if ($linkCount > 0) {

-                        $articleContent->appendChild($footnotesWrapper);           

-                }

-        }

-

-        /**

-        * Reverts P elements with class 'readability-styled'

-        * to text nodes - which is what they were before.

-        *

-        * @param DOMElement

-        * @return void

-        */

-        function revertReadabilityStyledElements($articleContent) {

-                $xpath = new DOMXPath($articleContent->ownerDocument);

-                $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);

-                //$elems = $articleContent->getElementsByTagName('p');

-                for ($i = $elems->length-1; $i >= 0; $i--) {

-                        $e = $elems->item($i);

-                        $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);

-                        //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {

-                        //      $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);

-                        //}

-                }

-        }

-        

-        /**

-        * Prepare the article node for display. Clean out any inline styles,

-        * iframes, forms, strip extraneous <p> tags, etc.

-        *

-        * @param DOMElement

-        * @return void

-        */

-        function prepArticle($articleContent) {

-                $this->cleanStyles($articleContent);

-                $this->killBreaks($articleContent);

-                if ($this->revertForcedParagraphElements) {

-                        $this->revertReadabilityStyledElements($articleContent);

-                }

-

-                /* Clean out junk from the article content */

-                $this->cleanConditionally($articleContent, 'form');

-                $this->clean($articleContent, 'object');

-                $this->clean($articleContent, 'h1');

-

-                /**

-                * If there is only one h2, they are probably using it

-                * as a header and not a subheader, so remove it since we already have a header.

-                ***/

-                if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {

-                        $this->clean($articleContent, 'h2'); 

-                }

-                $this->clean($articleContent, 'iframe');

-

-                $this->cleanHeaders($articleContent);

-

-                /* Do these last as the previous stuff may have removed junk that will affect these */

-                $this->cleanConditionally($articleContent, 'table');

-                $this->cleanConditionally($articleContent, 'ul');

-                $this->cleanConditionally($articleContent, 'div');

-

-                /* Remove extra paragraphs */

-                $articleParagraphs = $articleContent->getElementsByTagName('p');

-                for ($i = $articleParagraphs->length-1; $i >= 0; $i--)

-                {

-                        $imgCount    = $articleParagraphs->item($i)->getElementsByTagName('img')->length;

-                        $embedCount  = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;

-                        $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;

-                        $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;

-                        

-                        if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')

-                        {

-                                $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));

-                        }

-                }

-

-                try {

-                        $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);

-                        //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');      

-                }

-                catch (Exception $e) {

-                        $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);

-                }

-        }

-        

-        /**

-        * Initialize a node with the readability object. Also checks the

-        * className/id for special names to add to its score.

-        *

-        * @param Element

-        * @return void

-        **/

-        protected function initializeNode($node) {

-                $readability = $this->dom->createAttribute('readability');

-                $readability->value = 0; // this is our contentScore

-                $node->setAttributeNode($readability);                   

-

-                switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case

-                        case 'DIV':

-                                $readability->value += 5;

-                                break;

-

-                        case 'PRE':

-                        case 'TD':

-                        case 'BLOCKQUOTE':

-                                $readability->value += 3;

-                                break;

-                                

-                        case 'ADDRESS':

-                        case 'OL':

-                        case 'UL':

-                        case 'DL':

-                        case 'DD':

-                        case 'DT':

-                        case 'LI':

-                        case 'FORM':

-                                $readability->value -= 3;

-                                break;

-

-                        case 'H1':

-                        case 'H2':

-                        case 'H3':

-                        case 'H4':

-                        case 'H5':

-                        case 'H6':

-                        case 'TH':

-                                $readability->value -= 5;

-                                break;

-                }

-                $readability->value += $this->getClassWeight($node);

-        }

-        

-        /***

-        * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is

-        *               most likely to be the stuff a user wants to read. Then return it wrapped up in a div.

-        *

-        * @return DOMElement

-        **/

-        protected function grabArticle($page=null) {

-                $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);

-                if (!$page) $page = $this->dom;

-                $allElements = $page->getElementsByTagName('*');

-                /**

-                * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs

-                * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)

-                *

-                * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5

-                * TODO: Shouldn't this be a reverse traversal?

-                **/

-                $node = null;

-                $nodesToScore = array();

-                for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {

-                //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {

-                        //$node = $targetList->item($nodeIndex);

-                        $tagName = strtoupper($node->tagName);

-                        /* Remove unlikely candidates */

-                        if ($stripUnlikelyCandidates) {

-                                $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');

-                                if (

-                                        preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&

-                                        !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&

-                                        $tagName != 'BODY'

-                                )

-                                {

-                                        $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);

-                                        //$nodesToRemove[] = $node;

-                                        $node->parentNode->removeChild($node);

-                                        $nodeIndex--;

-                                        continue;

-                                }               

-                        }

-

-                        if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {

-                                $nodesToScore[] = $node;

-                        }

-

-                        /* Turn all divs that don't have children block level elements into p's */

-                        if ($tagName == 'DIV') {

-                                if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {

-                                        //$this->dbg('Altering div to p');

-                                        $newNode = $this->dom->createElement('p');

-                                        try {

-                                                $newNode->innerHTML = $node->innerHTML;

-                                                //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);

-                                                $node->parentNode->replaceChild($newNode, $node);

-                                                $nodeIndex--;

-                                                $nodesToScore[] = $node; // or $newNode?

-                                        }

-                                        catch(Exception $e) {

-                                                $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);

-                                        }

-                                }

-                                else

-                                {

-                                        /* EXPERIMENTAL */

-                                        // TODO: change these p elements back to text nodes after processing

-                                        for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {

-                                                $childNode = $node->childNodes->item($i);

-                                                if ($childNode->nodeType == 3) { // XML_TEXT_NODE

-                                                        //$this->dbg('replacing text node with a p tag with the same content.');

-                                                        $p = $this->dom->createElement('p');

-                                                        $p->innerHTML = $childNode->nodeValue;

-                                                        $p->setAttribute('style', 'display: inline;');

-                                                        $p->setAttribute('class', 'readability-styled');

-                                                        $childNode->parentNode->replaceChild($p, $childNode);

-                                                }

-                                        }

-                                }

-                        }

-                }

-                

-                /**

-                * Loop through all paragraphs, and assign a score to them based on how content-y they look.

-                * Then add their score to their parent node.

-                *

-                * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.

-                **/

-                $candidates = array();

-                for ($pt=0; $pt < count($nodesToScore); $pt++) {

-                        $parentNode      = $nodesToScore[$pt]->parentNode;

-                        // $grandParentNode = $parentNode ? $parentNode->parentNode : null;

-                        $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);

-                        $innerText       = $this->getInnerText($nodesToScore[$pt]);

-

-                        if (!$parentNode || !isset($parentNode->tagName)) {

-                                continue;

-                        }

-

-                        /* If this paragraph is less than 25 characters, don't even count it. */

-                        if(strlen($innerText) < 25) {

-                                continue;

-                        }

-

-                        /* Initialize readability data for the parent. */

-                        if (!$parentNode->hasAttribute('readability')) 

-                        {

-                                $this->initializeNode($parentNode);

-                                $candidates[] = $parentNode;

-                        }

-

-                        /* Initialize readability data for the grandparent. */

-                        if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))

-                        {

-                                $this->initializeNode($grandParentNode);

-                                $candidates[] = $grandParentNode;

-                        }

-

-                        $contentScore = 0;

-

-                        /* Add a point for the paragraph itself as a base. */

-                        $contentScore++;

-

-                        /* Add points for any commas within this paragraph */

-                        $contentScore += count(explode(',', $innerText));

-                        

-                        /* For every 100 characters in this paragraph, add another point. Up to 3 points. */

-                        $contentScore += min(floor(strlen($innerText) / 100), 3);

-                        

-                        /* Add the score to the parent. The grandparent gets half. */

-                        $parentNode->getAttributeNode('readability')->value += $contentScore;

-

-                        if ($grandParentNode) {

-                                $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;             

-                        }

-                }

-

-                /**

-                * After we've calculated scores, loop through all of the possible candidate nodes we found

-                * and find the one with the highest score.

-                **/

-                $topCandidate = null;

-                for ($c=0, $cl=count($candidates); $c < $cl; $c++)

-                {

-                        /**

-                        * Scale the final candidates score based on link density. Good content should have a

-                        * relatively small link density (5% or less) and be mostly unaffected by this operation.

-                        **/

-                        $readability = $candidates[$c]->getAttributeNode('readability');

-                        $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));

-

-                        $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);

-

-                        if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {

-                                $topCandidate = $candidates[$c];

-                        }

-                }

-

-                /**

-                * If we still have no top candidate, just use the body as a last resort.

-                * We also have to copy the body node so it is something we can modify.

-                **/

-                if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')

-                {

-                        $topCandidate = $this->dom->createElement('div');

-                        if ($page instanceof DOMDocument) {

-                                if (!isset($page->documentElement)) {

-                                        // we don't have a body either? what a mess! :)

-                                } else {

-                                        $topCandidate->innerHTML = $page->documentElement->innerHTML;

-                                        $page->documentElement->innerHTML = '';

-                                        $page->documentElement->appendChild($topCandidate);

-                                }

-                        } else {

-                                $topCandidate->innerHTML = $page->innerHTML;

-                                $page->innerHTML = '';

-                                $page->appendChild($topCandidate);

-                        }

-                        $this->initializeNode($topCandidate);

-                }

-

-                /**

-                * Now that we have the top candidate, look through its siblings for content that might also be related.

-                * Things like preambles, content split by ads that we removed, etc.

-                **/

-                $articleContent        = $this->dom->createElement('div');

-                $articleContent->setAttribute('id', 'readability-content');

-                $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);

-                $siblingNodes          = $topCandidate->parentNode->childNodes;

-                if (!isset($siblingNodes)) {

-                        $siblingNodes = new stdClass;

-                        $siblingNodes->length = 0;

-                }

-

-                for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)

-                {

-                        $siblingNode = $siblingNodes->item($s);

-                        $append      = false;

-

-                        $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));

-

-                        //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));

-

-                        if ($siblingNode === $topCandidate)

-                        // or if ($siblingNode->isSameNode($topCandidate))

-                        {

-                                $append = true;

-                        }

-

-                        $contentBonus = 0;

-                        /* Give a bonus if sibling nodes and top candidates have the example same classname */

-                        if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {

-                                $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;

-                        }

-

-                        if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)

-                        {

-                                $append = true;

-                        }

-                        

-                        if (strtoupper($siblingNode->nodeName) == 'P') {

-                                $linkDensity = $this->getLinkDensity($siblingNode);

-                                $nodeContent = $this->getInnerText($siblingNode);

-                                $nodeLength  = strlen($nodeContent);

-                                

-                                if ($nodeLength > 80 && $linkDensity < 0.25)

-                                {

-                                        $append = true;

-                                }

-                                else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))

-                                {

-                                        $append = true;

-                                }

-                        }

-

-                        if ($append)

-                        {

-                                $this->dbg('Appending node: ' . $siblingNode->nodeName);

-

-                                $nodeToAppend = null;

-                                $sibNodeName = strtoupper($siblingNode->nodeName);

-                                if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {

-                                        /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */

-                                        

-                                        $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');

-                                        $nodeToAppend = $this->dom->createElement('div');

-                                        try {

-                                                $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));

-                                                $nodeToAppend->innerHTML = $siblingNode->innerHTML;

-                                        }

-                                        catch(Exception $e)

-                                        {

-                                                $this->dbg('Could not alter siblingNode to div, reverting back to original.');

-                                                $nodeToAppend = $siblingNode;

-                                                $s--;

-                                                $sl--;

-                                        }

-                                } else {

-                                        $nodeToAppend = $siblingNode;

-                                        $s--;

-                                        $sl--;

-                                }

-                                

-                                /* To ensure a node does not interfere with readability styles, remove its classnames */

-                                $nodeToAppend->removeAttribute('class');

-

-                                /* Append sibling and subtract from our list because it removes the node when you append to another node */

-                                $articleContent->appendChild($nodeToAppend);

-                        }

-                }

-

-                /**

-                * So we have all of the content that we need. Now we clean it up for presentation.

-                **/

-                $this->prepArticle($articleContent);

-

-                /**

-                * Now that we've gone through the full algorithm, check to see if we got any meaningful content.

-                * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher

-                * likelihood of finding the content, and the sieve approach gives us a higher likelihood of

-                * finding the -right- content.

-                **/

-                if (strlen($this->getInnerText($articleContent, false)) < 250)

-                {

-                        // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7

-                        // in the meantime, we check and create an empty element if it's not there.

-                        if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');

-                        $this->body->innerHTML = $this->bodyCache;

-                        

-                        if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {

-                                $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);

-                                return $this->grabArticle($this->body);

-                        }

-                        else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {

-                                $this->removeFlag(self::FLAG_WEIGHT_CLASSES);

-                                return $this->grabArticle($this->body);              

-                        }

-                        else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {

-                                $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);

-                                return $this->grabArticle($this->body);

-                        }

-                        else {

-                                return false;

-                        }

-                }

-                return $articleContent;

-        }

-        

-        /**

-        * Remove script tags from document

-        *

-        * @param DOMElement

-        * @return void

-        */

-        public function removeScripts($doc) {

-                $scripts = $doc->getElementsByTagName('script');

-                for($i = $scripts->length-1; $i >= 0; $i--)

-                {

-                        $scripts->item($i)->parentNode->removeChild($scripts->item($i));

-                }

-        }

-        

-        /**

-        * Get the inner text of a node.

-        * This also strips out any excess whitespace to be found.

-        *

-        * @param DOMElement $

-        * @param boolean $normalizeSpaces (default: true)

-        * @return string

-        **/

-        public function getInnerText($e, $normalizeSpaces=true) {

-                $textContent = '';

-

-                if (!isset($e->textContent) || $e->textContent == '') {

-                        return '';

-                }

-

-                $textContent = trim($e->textContent);

-

-                if ($normalizeSpaces) {

-                        return preg_replace($this->regexps['normalize'], ' ', $textContent);

-                } else {

-                        return $textContent;

-                }

-        }

-

-        /**

-        * Get the number of times a string $s appears in the node $e.

-        *

-        * @param DOMElement $e

-        * @param string - what to count. Default is ","

-        * @return number (integer)

-        **/

-        public function getCharCount($e, $s=',') {

-                return substr_count($this->getInnerText($e), $s);

-        }

-

-        /**

-        * Remove the style attribute on every $e and under.

-        *

-        * @param DOMElement $e

-        * @return void

-        */

-        public function cleanStyles($e) {

-                if (!is_object($e)) return;

-                $elems = $e->getElementsByTagName('*');

-                foreach ($elems as $elem) {

-                        $elem->removeAttribute('style');

-                }

-        }

-        

-        /**

-        * Get the density of links as a percentage of the content

-        * This is the amount of text that is inside a link divided by the total text in the node.

-        * 

-        * @param DOMElement $e

-        * @return number (float)

-        */

-        public function getLinkDensity($e) {

-                $links      = $e->getElementsByTagName('a');

-                $textLength = strlen($this->getInnerText($e));

-                $linkLength = 0;

-                for ($i=0, $il=$links->length; $i < $il; $i++)

-                {

-                        $linkLength += strlen($this->getInnerText($links->item($i)));

-                }

-                if ($textLength > 0) {

-                        return $linkLength / $textLength;

-                } else {

-                        return 0;

-                }

-        }

-        

-        /**

-        * Get an elements class/id weight. Uses regular expressions to tell if this 

-        * element looks good or bad.

-        *

-        * @param DOMElement $e

-        * @return number (Integer)

-        */

-        public function getClassWeight($e) {

-                if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {

-                        return 0;

-                }

-

-                $weight = 0;

-

-                /* Look for a special classname */

-                if ($e->hasAttribute('class') && $e->getAttribute('class') != '')

-                {

-                        if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {

-                                $weight -= 25;

-                        }

-                        if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {

-                                $weight += 25;

-                        }

-                }

-

-                /* Look for a special ID */

-                if ($e->hasAttribute('id') && $e->getAttribute('id') != '')

-                {

-                        if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {

-                                $weight -= 25;

-                        }

-                        if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {

-                                $weight += 25;

-                        }

-                }

-                return $weight;

-        }

-

-        /**

-        * Remove extraneous break tags from a node.

-        *

-        * @param DOMElement $node

-        * @return void

-        */

-        public function killBreaks($node) {

-                $html = $node->innerHTML;

-                $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);

-                $node->innerHTML = $html;

-        }

-

-        /**

-        * Clean a node of all elements of type "tag".

-        * (Unless it's a youtube/vimeo video. People love movies.)

-        *

-        * Updated 2012-09-18 to preserve youtube/vimeo iframes

-        *

-        * @param DOMElement $e

-        * @param string $tag

-        * @return void

-        */

-        public function clean($e, $tag) {

-                $targetList = $e->getElementsByTagName($tag);

-                $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');

-                

-                for ($y=$targetList->length-1; $y >= 0; $y--) {

-                        /* Allow youtube and vimeo videos through as people usually want to see those. */

-                        if ($isEmbed) {

-                                $attributeValues = '';

-                                for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {

-                                        $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)

-                                }

-                                

-                                /* First, check the elements attributes to see if any of them contain youtube or vimeo */

-                                if (preg_match($this->regexps['video'], $attributeValues)) {

-                                        continue;

-                                }

-

-                                /* Then check the elements inside this element for the same. */

-                                if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {

-                                        continue;

-                                }

-                        }

-                        $targetList->item($y)->parentNode->removeChild($targetList->item($y));

-                }

-        }

-        

-        /**

-        * Clean an element of all tags of type "tag" if they look fishy.

-        * "Fishy" is an algorithm based on content length, classnames, 

-        * link density, number of images & embeds, etc.

-        *

-        * @param DOMElement $e

-        * @param string $tag

-        * @return void

-        */

-        public function cleanConditionally($e, $tag) {

-                if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {

-                        return;

-                }

-

-                $tagsList = $e->getElementsByTagName($tag);

-                $curTagsLength = $tagsList->length;

-

-                /**

-                * Gather counts for other typical elements embedded within.

-                * Traverse backwards so we can remove nodes at the same time without effecting the traversal.

-                *

-                * TODO: Consider taking into account original contentScore here.

-                */

-                for ($i=$curTagsLength-1; $i >= 0; $i--) {

-                        $weight = $this->getClassWeight($tagsList->item($i));

-                        $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;

-                        

-                        $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));

-

-                        if ($weight + $contentScore < 0) {

-                                $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));

-                        }

-                        else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {

-                                /**

-                                * If there are not very many commas, and the number of

-                                * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.

-                                **/

-                                $p      = $tagsList->item($i)->getElementsByTagName('p')->length;

-                                $img    = $tagsList->item($i)->getElementsByTagName('img')->length;

-                                $li     = $tagsList->item($i)->getElementsByTagName('li')->length-100;

-                                $input  = $tagsList->item($i)->getElementsByTagName('input')->length;

-                                $a              = $tagsList->item($i)->getElementsByTagName('a')->length;

-

-                                $embedCount = 0;

-                                $embeds = $tagsList->item($i)->getElementsByTagName('embed');

-                                for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {

-                                        if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {

-                                                $embedCount++; 

-                                        }

-                                }

-                                $embeds = $tagsList->item($i)->getElementsByTagName('iframe');

-                                for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {

-                                        if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {

-                                                $embedCount++; 

-                                        }

-                                }

-

-                                $linkDensity   = $this->getLinkDensity($tagsList->item($i));

-                                $contentLength = strlen($this->getInnerText($tagsList->item($i)));

-                                $toRemove      = false;

-

-                                if ($this->lightClean) {

-                                        $this->dbg('Light clean...');

-                                        if ( ($img > $p) && ($img > 4) ) {

-                                                $this->dbg(' more than 4 images and more image elements than paragraph elements');

-                                                $toRemove = true;

-                                        } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {

-                                                $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');

-                                                $toRemove = true;

-                                        } else if ( $input > floor($p/3) ) {

-                                                $this->dbg(' too many <input> elements');

-                                                $toRemove = true; 

-                                        } else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {

-                                                $this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');

-                                                $toRemove = true;

-                                        } else if($weight < 25 && $linkDensity > 0.2) {

-                                                $this->dbg(' weight smaller than 25 and link density above 0.2');

-                                                $toRemove = true;

-                                        } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {

-                                                $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');

-                                                $toRemove = true;

-                                        } else if($embedCount > 3) {

-                                                $this->dbg(' more than 3 embeds');

-                                                $toRemove = true;

-                                        }

-                                } else {

-                                        $this->dbg('Standard clean...');

-                                        if ( $img > $p ) {

-                                                $this->dbg(' more image elements than paragraph elements');

-                                                $toRemove = true;

-                                        } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {

-                                                $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');

-                                                $toRemove = true;

-                                        } else if ( $input > floor($p/3) ) {

-                                                $this->dbg(' too many <input> elements');

-                                                $toRemove = true; 

-                                        } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {

-                                                $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');

-                                                $toRemove = true;

-                                        } else if($weight < 25 && $linkDensity > 0.2) {

-                                                $this->dbg(' weight smaller than 25 and link density above 0.2');

-                                                $toRemove = true;

-                                        } else if($weight >= 25 && $linkDensity > 0.5) {

-                                                $this->dbg(' weight above 25 but link density greater than 0.5');

-                                                $toRemove = true;

-                                        } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {

-                                                $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');

-                                                $toRemove = true;

-                                        }

-                                }

-

-                                if ($toRemove) {

-                                        //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);

-                                        $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));

-                                }

-                        }

-                }

-        }

-

-        /**

-        * Clean out spurious headers from an Element. Checks things like classnames and link density.

-        *

-        * @param DOMElement $e

-        * @return void

-        */

-        public function cleanHeaders($e) {

-                for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {

-                        $headers = $e->getElementsByTagName('h' . $headerIndex);

-                        for ($i=$headers->length-1; $i >=0; $i--) {

-                                if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {

-                                        $headers->item($i)->parentNode->removeChild($headers->item($i));

-                                }

-                        }

-                }

-        }

-

-        public function flagIsActive($flag) {

-                return ($this->flags & $flag) > 0;

-        }

-        

-        public function addFlag($flag) {

-                $this->flags = $this->flags | $flag;

-        }

-        

-        public function removeFlag($flag) {

-                $this->flags = $this->flags & ~$flag;

-        }

-}

+<?php
+/** 
+* Arc90's Readability ported to PHP for FiveFilters.org
+* Based on readability.js version 1.7.1 (without multi-page support)
+* Updated to allow HTML5 parsing with html5lib
+* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
+* ------------------------------------------------------
+* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
+* Arc90's project URL: http://lab.arc90.com/experiments/readability/
+* JS Source: http://code.google.com/p/arc90labs-readability
+* Ported by: Keyvan Minoukadeh, http://www.keyvan.net
+* More information: http://fivefilters.org/content-only/
+* License: Apache License, Version 2.0
+* Requires: PHP5
+* Date: 2012-09-19
+* 
+* Differences between the PHP port and the original
+* ------------------------------------------------------
+* Arc90's Readability is designed to run in the browser. It works on the DOM 
+* tree (the parsed HTML) after the page's CSS styles have been applied and 
+* Javascript code executed. This PHP port does not run inside a browser. 
+* We use PHP's ability to parse HTML to build our DOM tree, but we cannot 
+* rely on CSS or Javascript support. As such, the results will not always 
+* match Arc90's Readability. (For example, if a web page contains CSS style 
+* rules or Javascript code which hide certain HTML elements from display, 
+* Arc90's Readability will dismiss those from consideration but our PHP port, 
+* unable to understand CSS or Javascript, will not know any better.)
+* 
+* Another significant difference is that the aim of Arc90's Readability is 
+* to re-present the main content block of a given web page so users can 
+* read it more easily in their browsers. Correct identification, clean up, 
+* and separation of the content block is only a part of this process. 
+* This PHP port is only concerned with this part, it does not include code 
+* that relates to presentation in the browser - Arc90 already do 
+* that extremely well, and for PDF output there's FiveFilters.org's 
+* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
+* 
+* Finally, this class contains methods that might be useful for developers 
+* working on HTML document fragments. So without deviating too much from 
+* the original code (which I don't want to do because it makes debugging 
+* and updating more difficult), I've tried to make it a little more 
+* developer friendly. You should be able to use the methods here on 
+* existing DOMElement objects without passing an entire HTML document to 
+* be parsed.
+*/
+
+// This class allows us to do JavaScript like assignements to innerHTML
+require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
+
+// Alternative usage (for testing only!)
+// uncomment the lines below and call Readability.php in your browser 
+// passing it the URL of the page you'd like content from, e.g.:
+// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
+
+/*
+if (!isset($_GET['url']) || $_GET['url'] == '') {
+        die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
+}
+$url = $_GET['url'];
+if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
+$html = file_get_contents($url);
+$r = new Readability($html, $url);
+$r->init();
+echo $r->articleContent->innerHTML;
+*/
+
+class Readability
+{
+        public $version = '1.7.1-without-multi-page';
+        public $convertLinksToFootnotes = false;
+        public $revertForcedParagraphElements = true;
+        public $articleTitle;
+        public $articleContent;
+        public $dom;
+        public $url = null; // optional - URL where HTML was retrieved
+        public $debug = false;
+        public $lightClean = true; // preserves more content (experimental) added 2012-09-19
+        protected $body = null; // 
+        protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
+        protected $flags = 7; // 1 | 2 | 4;   // Start with all flags set.
+        protected $success = false; // indicates whether we were able to extract or not
+        
+        /**
+        * All of the regular expressions in use within readability.
+        * Defined up here so we don't instantiate them repeatedly in loops.
+        **/
+        public $regexps = array(
+                'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
+                'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
+                'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
+                'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
+                'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
+                'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
+                'replaceFonts' => '/<(\/?)font[^>]*>/i',
+                // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
+                'normalize' => '/\s{2,}/',
+                'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
+                'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
+                'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
+        );      
+        
+        /* constants */
+        const FLAG_STRIP_UNLIKELYS = 1;
+        const FLAG_WEIGHT_CLASSES = 2;
+        const FLAG_CLEAN_CONDITIONALLY = 4;
+        
+        /**
+        * Create instance of Readability
+        * @param string UTF-8 encoded string
+        * @param string (optional) URL associated with HTML (used for footnotes)
+        * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
+        */      
+        function __construct($html, $url=null, $parser='libxml')
+        {
+                $this->url = $url;
+                /* Turn all double br's into p's */
+                $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
+                $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
+                $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
+                if (trim($html) == '') $html = '<html></html>';
+                if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
+                        // all good
+                } else {
+                        $this->dom = new DOMDocument();
+                        $this->dom->preserveWhiteSpace = false;
+                        @$this->dom->loadHTML($html);
+                }
+                $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
+        }
+
+        /**
+        * Get article title element
+        * @return DOMElement
+        */
+        public function getTitle() {
+                return $this->articleTitle;
+        }
+        
+        /**
+        * Get article content element
+        * @return DOMElement
+        */
+        public function getContent() {
+                return $this->articleContent;
+        }       
+        
+        /**
+        * Runs readability.
+        * 
+        * Workflow:
+        *  1. Prep the document by removing script tags, css, etc.
+        *  2. Build readability's DOM tree.
+        *  3. Grab the article content from the current dom tree.
+        *  4. Replace the current DOM tree with the new one.
+        *  5. Read peacefully.
+        *
+        * @return boolean true if we found content, false otherwise
+        **/
+        public function init()
+        {
+                if (!isset($this->dom->documentElement)) return false;
+                $this->removeScripts($this->dom);
+                //die($this->getInnerHTML($this->dom->documentElement));
+                
+                // Assume successful outcome
+                $this->success = true;
+
+                $bodyElems = $this->dom->getElementsByTagName('body');
+                if ($bodyElems->length > 0) {
+                        if ($this->bodyCache == null) {
+                                $this->bodyCache = $bodyElems->item(0)->innerHTML;
+                        }
+                        if ($this->body == null) {
+                                $this->body = $bodyElems->item(0);
+                        }
+                }
+
+                $this->prepDocument();
+                
+                //die($this->dom->documentElement->parentNode->nodeType);
+                //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
+                //die($this->getInnerHTML($this->dom->documentElement));
+
+                /* Build readability's DOM tree */
+                $overlay        = $this->dom->createElement('div');
+                $innerDiv       = $this->dom->createElement('div');
+                $articleTitle   = $this->getArticleTitle();
+                $articleContent = $this->grabArticle();
+
+                if (!$articleContent) {
+                        $this->success = false;
+                        $articleContent = $this->dom->createElement('div');
+                        $articleContent->setAttribute('id', 'readability-content');
+                        $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';            
+                }
+                
+                $overlay->setAttribute('id', 'readOverlay');
+                $innerDiv->setAttribute('id', 'readInner');
+
+                /* Glue the structure of our document together. */
+                $innerDiv->appendChild($articleTitle);
+                $innerDiv->appendChild($articleContent);
+                $overlay->appendChild($innerDiv);
+                
+                /* Clear the old HTML, insert the new content. */
+                $this->body->innerHTML = '';
+                $this->body->appendChild($overlay);
+                //document.body.insertBefore(overlay, document.body.firstChild);
+                $this->body->removeAttribute('style');
+
+                $this->postProcessContent($articleContent);
+                
+                // Set title and content instance variables
+                $this->articleTitle = $articleTitle;
+                $this->articleContent = $articleContent;
+                
+                return $this->success;
+        }
+        
+        /**
+        * Debug
+        */
+        protected function dbg($msg) {
+                if ($this->debug) echo '* ',$msg, "\n";
+        }
+        
+        /**
+        * Run any post-process modifications to article content as necessary.
+        *
+        * @param DOMElement
+        * @return void
+        */
+        public function postProcessContent($articleContent) {
+                if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { 
+                        $this->addFootnotes($articleContent);
+                }
+        }
+        
+        /**
+        * Get the article title as an H1.
+        *
+        * @return DOMElement
+        */
+        protected function getArticleTitle() {
+                $curTitle = '';
+                $origTitle = '';
+
+                try {
+                        $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
+                } catch(Exception $e) {}
+                
+                if (preg_match('/ [\|\-] /', $curTitle))
+                {
+                        $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
+                        
+                        if (count(explode(' ', $curTitle)) < 3) {
+                                $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
+                        }
+                }
+                else if (strpos($curTitle, ': ') !== false)
+                {
+                        $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
+
+                        if (count(explode(' ', $curTitle)) < 3) {
+                                $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
+                        }
+                }
+                else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)
+                {
+                        $hOnes = $this->dom->getElementsByTagName('h1');
+                        if($hOnes->length == 1)
+                        {
+                                $curTitle = $this->getInnerText($hOnes->item(0));
+                        }
+                }
+
+                $curTitle = trim($curTitle);
+
+                if (count(explode(' ', $curTitle)) <= 4) {
+                        $curTitle = $origTitle;
+                }
+                
+                $articleTitle = $this->dom->createElement('h1');
+                $articleTitle->innerHTML = $curTitle;
+                
+                return $articleTitle;
+        }
+        
+        /**
+        * Prepare the HTML document for readability to scrape it.
+        * This includes things like stripping javascript, CSS, and handling terrible markup.
+        * 
+        * @return void
+        **/
+        protected function prepDocument() {
+                /**
+                * In some cases a body element can't be found (if the HTML is totally hosed for example)
+                * so we create a new body node and append it to the document.
+                */
+                if ($this->body == null)
+                {
+                        $this->body = $this->dom->createElement('body');
+                        $this->dom->documentElement->appendChild($this->body);
+                }
+                $this->body->setAttribute('id', 'readabilityBody');
+
+                /* Remove all style tags in head */
+                $styleTags = $this->dom->getElementsByTagName('style');
+                for ($i = $styleTags->length-1; $i >= 0; $i--)
+                {
+                        $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
+                }
+
+                /* Turn all double br's into p's */
+                /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
+                //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
+                // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
+                // Manipulating innerHTML as it's done in JS is not possible in PHP.
+        }
+
+        /**
+        * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
+        * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
+        *
+        * @return void
+        **/
+        public function addFootnotes($articleContent) {
+                $footnotesWrapper = $this->dom->createElement('div');
+                $footnotesWrapper->setAttribute('id', 'readability-footnotes');
+                $footnotesWrapper->innerHTML = '<h3>References</h3>';
+                
+                $articleFootnotes = $this->dom->createElement('ol');
+                $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
+                $footnotesWrapper->appendChild($articleFootnotes);
+                
+                $articleLinks = $articleContent->getElementsByTagName('a');
+                
+                $linkCount = 0;
+                for ($i = 0; $i < $articleLinks->length; $i++)
+                {
+                        $articleLink  = $articleLinks->item($i);
+                        $footnoteLink = $articleLink->cloneNode(true);
+                        $refLink      = $this->dom->createElement('a');
+                        $footnote     = $this->dom->createElement('li');
+                        $linkDomain   = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
+                        if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);
+                        //linkDomain   = footnoteLink.host ? footnoteLink.host : document.location.host,
+                        $linkText     = $this->getInnerText($articleLink);
+                        
+                        if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
+                                continue;
+                        }
+                        
+                        $linkCount++;
+
+                        /** Add a superscript reference after the article link */
+                        $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
+                        $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
+                        $refLink->setAttribute('class', 'readability-DoNotFootnote');
+                        $refLink->setAttribute('style', 'color: inherit;');
+                        
+                        //TODO: does this work or should we use DOMNode.isSameNode()?
+                        if ($articleLink->parentNode->lastChild == $articleLink) {
+                                $articleLink->parentNode->appendChild($refLink);
+                        } else {
+                                $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
+                        }
+
+                        $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
+                        $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
+
+                        $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
+
+                        $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);
+                        $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
+                        
+                        $footnote->appendChild($footnoteLink);
+                        if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';
+                        
+                        $articleFootnotes->appendChild($footnote);
+                }
+
+                if ($linkCount > 0) {
+                        $articleContent->appendChild($footnotesWrapper);           
+                }
+        }
+
+        /**
+        * Reverts P elements with class 'readability-styled'
+        * to text nodes - which is what they were before.
+        *
+        * @param DOMElement
+        * @return void
+        */
+        function revertReadabilityStyledElements($articleContent) {
+                $xpath = new DOMXPath($articleContent->ownerDocument);
+                $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
+                //$elems = $articleContent->getElementsByTagName('p');
+                for ($i = $elems->length-1; $i >= 0; $i--) {
+                        $e = $elems->item($i);
+                        $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
+                        //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
+                        //      $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
+                        //}
+                }
+        }
+        
+        /**
+        * Prepare the article node for display. Clean out any inline styles,
+        * iframes, forms, strip extraneous <p> tags, etc.
+        *
+        * @param DOMElement
+        * @return void
+        */
+        function prepArticle($articleContent) {
+                $this->cleanStyles($articleContent);
+                $this->killBreaks($articleContent);
+                if ($this->revertForcedParagraphElements) {
+                        $this->revertReadabilityStyledElements($articleContent);
+                }
+
+                /* Clean out junk from the article content */
+                $this->cleanConditionally($articleContent, 'form');
+                $this->clean($articleContent, 'object');
+                $this->clean($articleContent, 'h1');
+
+                /**
+                * If there is only one h2, they are probably using it
+                * as a header and not a subheader, so remove it since we already have a header.
+                ***/
+                if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
+                        $this->clean($articleContent, 'h2'); 
+                }
+                $this->clean($articleContent, 'iframe');
+
+                $this->cleanHeaders($articleContent);
+
+                /* Do these last as the previous stuff may have removed junk that will affect these */
+                $this->cleanConditionally($articleContent, 'table');
+                $this->cleanConditionally($articleContent, 'ul');
+                $this->cleanConditionally($articleContent, 'div');
+
+                /* Remove extra paragraphs */
+                $articleParagraphs = $articleContent->getElementsByTagName('p');
+                for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
+                {
+                        $imgCount    = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
+                        $embedCount  = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
+                        $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
+                        $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
+                        
+                        if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
+                        {
+                                $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
+                        }
+                }
+
+                try {
+                        $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);
+                        //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');      
+                }
+                catch (Exception $e) {
+                        $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
+                }
+        }
+        
+        /**
+        * Initialize a node with the readability object. Also checks the
+        * className/id for special names to add to its score.
+        *
+        * @param Element
+        * @return void
+        **/
+        protected function initializeNode($node) {
+                $readability = $this->dom->createAttribute('readability');
+                $readability->value = 0; // this is our contentScore
+                $node->setAttributeNode($readability);                   
+
+                switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
+                        case 'DIV':
+                                $readability->value += 5;
+                                break;
+
+                        case 'PRE':
+                        case 'TD':
+                        case 'BLOCKQUOTE':
+                                $readability->value += 3;
+                                break;
+                                
+                        case 'ADDRESS':
+                        case 'OL':
+                        case 'UL':
+                        case 'DL':
+                        case 'DD':
+                        case 'DT':
+                        case 'LI':
+                        case 'FORM':
+                                $readability->value -= 3;
+                                break;
+
+                        case 'H1':
+                        case 'H2':
+                        case 'H3':
+                        case 'H4':
+                        case 'H5':
+                        case 'H6':
+                        case 'TH':
+                                $readability->value -= 5;
+                                break;
+                }
+                $readability->value += $this->getClassWeight($node);
+        }
+        
+        /***
+        * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
+        *               most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
+        *
+        * @return DOMElement
+        **/
+        protected function grabArticle($page=null) {
+                $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
+                if (!$page) $page = $this->dom;
+                $allElements = $page->getElementsByTagName('*');
+                /**
+                * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
+                * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
+                *
+                * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
+                * TODO: Shouldn't this be a reverse traversal?
+                **/
+                $node = null;
+                $nodesToScore = array();
+                for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
+                //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
+                        //$node = $targetList->item($nodeIndex);
+                        $tagName = strtoupper($node->tagName);
+                        /* Remove unlikely candidates */
+                        if ($stripUnlikelyCandidates) {
+                                $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
+                                if (
+                                        preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
+                                        !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
+                                        $tagName != 'BODY'
+                                )
+                                {
+                                        $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
+                                        //$nodesToRemove[] = $node;
+                                        $node->parentNode->removeChild($node);
+                                        $nodeIndex--;
+                                        continue;
+                                }               
+                        }
+
+                        if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
+                                $nodesToScore[] = $node;
+                        }
+
+                        /* Turn all divs that don't have children block level elements into p's */
+                        if ($tagName == 'DIV') {
+                                if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
+                                        //$this->dbg('Altering div to p');
+                                        $newNode = $this->dom->createElement('p');
+                                        try {
+                                                $newNode->innerHTML = $node->innerHTML;
+                                                //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
+                                                $node->parentNode->replaceChild($newNode, $node);
+                                                $nodeIndex--;
+                                                $nodesToScore[] = $node; // or $newNode?
+                                        }
+                                        catch(Exception $e) {
+                                                $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
+                                        }
+                                }
+                                else
+                                {
+                                        /* EXPERIMENTAL */
+                                        // TODO: change these p elements back to text nodes after processing
+                                        for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
+                                                $childNode = $node->childNodes->item($i);
+                                                if ($childNode->nodeType == 3) { // XML_TEXT_NODE
+                                                        //$this->dbg('replacing text node with a p tag with the same content.');
+                                                        $p = $this->dom->createElement('p');
+                                                        $p->innerHTML = $childNode->nodeValue;
+                                                        $p->setAttribute('style', 'display: inline;');
+                                                        $p->setAttribute('class', 'readability-styled');
+                                                        $childNode->parentNode->replaceChild($p, $childNode);
+                                                }
+                                        }
+                                }
+                        }
+                }
+                
+                /**
+                * Loop through all paragraphs, and assign a score to them based on how content-y they look.
+                * Then add their score to their parent node.
+                *
+                * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
+                **/
+                $candidates = array();
+                for ($pt=0; $pt < count($nodesToScore); $pt++) {
+                        $parentNode      = $nodesToScore[$pt]->parentNode;
+                        // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
+                        $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
+                        $innerText       = $this->getInnerText($nodesToScore[$pt]);
+
+                        if (!$parentNode || !isset($parentNode->tagName)) {
+                                continue;
+                        }
+
+                        /* If this paragraph is less than 25 characters, don't even count it. */
+                        if(strlen($innerText) < 25) {
+                                continue;
+                        }
+
+                        /* Initialize readability data for the parent. */
+                        if (!$parentNode->hasAttribute('readability')) 
+                        {
+                                $this->initializeNode($parentNode);
+                                $candidates[] = $parentNode;
+                        }
+
+                        /* Initialize readability data for the grandparent. */
+                        if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
+                        {
+                                $this->initializeNode($grandParentNode);
+                                $candidates[] = $grandParentNode;
+                        }
+
+                        $contentScore = 0;
+
+                        /* Add a point for the paragraph itself as a base. */
+                        $contentScore++;
+
+                        /* Add points for any commas within this paragraph */
+                        $contentScore += count(explode(',', $innerText));
+                        
+                        /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
+                        $contentScore += min(floor(strlen($innerText) / 100), 3);
+                        
+                        /* Add the score to the parent. The grandparent gets half. */
+                        $parentNode->getAttributeNode('readability')->value += $contentScore;
+
+                        if ($grandParentNode) {
+                                $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;             
+                        }
+                }
+
+                /**
+                * After we've calculated scores, loop through all of the possible candidate nodes we found
+                * and find the one with the highest score.
+                **/
+                $topCandidate = null;
+                for ($c=0, $cl=count($candidates); $c < $cl; $c++)
+                {
+                        /**
+                        * Scale the final candidates score based on link density. Good content should have a
+                        * relatively small link density (5% or less) and be mostly unaffected by this operation.
+                        **/
+                        $readability = $candidates[$c]->getAttributeNode('readability');
+                        $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
+
+                        $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
+
+                        if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
+                                $topCandidate = $candidates[$c];
+                        }
+                }
+
+                /**
+                * If we still have no top candidate, just use the body as a last resort.
+                * We also have to copy the body node so it is something we can modify.
+                **/
+                if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
+                {
+                        $topCandidate = $this->dom->createElement('div');
+                        if ($page instanceof DOMDocument) {
+                                if (!isset($page->documentElement)) {
+                                        // we don't have a body either? what a mess! :)
+                                } else {
+                                        $topCandidate->innerHTML = $page->documentElement->innerHTML;
+                                        $page->documentElement->innerHTML = '';
+                                        $page->documentElement->appendChild($topCandidate);
+                                }
+                        } else {
+                                $topCandidate->innerHTML = $page->innerHTML;
+                                $page->innerHTML = '';
+                                $page->appendChild($topCandidate);
+                        }
+                        $this->initializeNode($topCandidate);
+                }
+
+                /**
+                * Now that we have the top candidate, look through its siblings for content that might also be related.
+                * Things like preambles, content split by ads that we removed, etc.
+                **/
+                $articleContent        = $this->dom->createElement('div');
+                $articleContent->setAttribute('id', 'readability-content');
+                $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
+                $siblingNodes          = $topCandidate->parentNode->childNodes;
+                if (!isset($siblingNodes)) {
+                        $siblingNodes = new stdClass;
+                        $siblingNodes->length = 0;
+                }
+
+                for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
+                {
+                        $siblingNode = $siblingNodes->item($s);
+                        $append      = false;
+
+                        $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
+
+                        //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
+
+                        if ($siblingNode === $topCandidate)
+                        // or if ($siblingNode->isSameNode($topCandidate))
+                        {
+                                $append = true;
+                        }
+
+                        $contentBonus = 0;
+                        /* Give a bonus if sibling nodes and top candidates have the example same classname */
+                        if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
+                                $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
+                        }
+
+                        if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
+                        {
+                                $append = true;
+                        }
+                        
+                        if (strtoupper($siblingNode->nodeName) == 'P') {
+                                $linkDensity = $this->getLinkDensity($siblingNode);
+                                $nodeContent = $this->getInnerText($siblingNode);
+                                $nodeLength  = strlen($nodeContent);
+                                
+                                if ($nodeLength > 80 && $linkDensity < 0.25)
+                                {
+                                        $append = true;
+                                }
+                                else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
+                                {
+                                        $append = true;
+                                }
+                        }
+
+                        if ($append)
+                        {
+                                $this->dbg('Appending node: ' . $siblingNode->nodeName);
+
+                                $nodeToAppend = null;
+                                $sibNodeName = strtoupper($siblingNode->nodeName);
+                                if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
+                                        /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
+                                        
+                                        $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
+                                        $nodeToAppend = $this->dom->createElement('div');
+                                        try {
+                                                $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
+                                                $nodeToAppend->innerHTML = $siblingNode->innerHTML;
+                                        }
+                                        catch(Exception $e)
+                                        {
+                                                $this->dbg('Could not alter siblingNode to div, reverting back to original.');
+                                                $nodeToAppend = $siblingNode;
+                                                $s--;
+                                                $sl--;
+                                        }
+                                } else {
+                                        $nodeToAppend = $siblingNode;
+                                        $s--;
+                                        $sl--;
+                                }
+                                
+                                /* To ensure a node does not interfere with readability styles, remove its classnames */
+                                $nodeToAppend->removeAttribute('class');
+
+                                /* Append sibling and subtract from our list because it removes the node when you append to another node */
+                                $articleContent->appendChild($nodeToAppend);
+                        }
+                }
+
+                /**
+                * So we have all of the content that we need. Now we clean it up for presentation.
+                **/
+                $this->prepArticle($articleContent);
+
+                /**
+                * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
+                * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
+                * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
+                * finding the -right- content.
+                **/
+                if (strlen($this->getInnerText($articleContent, false)) < 250)
+                {
+                        // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
+                        // in the meantime, we check and create an empty element if it's not there.
+                        if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');
+                        $this->body->innerHTML = $this->bodyCache;
+                        
+                        if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
+                                $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
+                                return $this->grabArticle($this->body);
+                        }
+                        else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
+                                $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
+                                return $this->grabArticle($this->body);              
+                        }
+                        else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
+                                $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
+                                return $this->grabArticle($this->body);
+                        }
+                        else {
+                                return false;
+                        }
+                }
+                return $articleContent;
+        }
+        
+        /**
+        * Remove script tags from document
+        *
+        * @param DOMElement
+        * @return void
+        */
+        public function removeScripts($doc) {
+                $scripts = $doc->getElementsByTagName('script');
+                for($i = $scripts->length-1; $i >= 0; $i--)
+                {
+                        $scripts->item($i)->parentNode->removeChild($scripts->item($i));
+                }
+        }
+        
+        /**
+        * Get the inner text of a node.
+        * This also strips out any excess whitespace to be found.
+        *
+        * @param DOMElement $
+        * @param boolean $normalizeSpaces (default: true)
+        * @return string
+        **/
+        public function getInnerText($e, $normalizeSpaces=true) {
+                $textContent = '';
+
+                if (!isset($e->textContent) || $e->textContent == '') {
+                        return '';
+                }
+
+                $textContent = trim($e->textContent);
+
+                if ($normalizeSpaces) {
+                        return preg_replace($this->regexps['normalize'], ' ', $textContent);
+                } else {
+                        return $textContent;
+                }
+        }
+
+        /**
+        * Get the number of times a string $s appears in the node $e.
+        *
+        * @param DOMElement $e
+        * @param string - what to count. Default is ","
+        * @return number (integer)
+        **/
+        public function getCharCount($e, $s=',') {
+                return substr_count($this->getInnerText($e), $s);
+        }
+
+        /**
+        * Remove the style attribute on every $e and under.
+        *
+        * @param DOMElement $e
+        * @return void
+        */
+        public function cleanStyles($e) {
+                if (!is_object($e)) return;
+                $elems = $e->getElementsByTagName('*');
+                foreach ($elems as $elem) {
+                        $elem->removeAttribute('style');
+                }
+        }
+        
+        /**
+        * Get the density of links as a percentage of the content
+        * This is the amount of text that is inside a link divided by the total text in the node.
+        * 
+        * @param DOMElement $e
+        * @return number (float)
+        */
+        public function getLinkDensity($e) {
+                $links      = $e->getElementsByTagName('a');
+                $textLength = strlen($this->getInnerText($e));
+                $linkLength = 0;
+                for ($i=0, $il=$links->length; $i < $il; $i++)
+                {
+                        $linkLength += strlen($this->getInnerText($links->item($i)));
+                }
+                if ($textLength > 0) {
+                        return $linkLength / $textLength;
+                } else {
+                        return 0;
+                }
+        }
+        
+        /**
+        * Get an elements class/id weight. Uses regular expressions to tell if this 
+        * element looks good or bad.
+        *
+        * @param DOMElement $e
+        * @return number (Integer)
+        */
+        public function getClassWeight($e) {
+                if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
+                        return 0;
+                }
+
+                $weight = 0;
+
+                /* Look for a special classname */
+                if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
+                {
+                        if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
+                                $weight -= 25;
+                        }
+                        if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
+                                $weight += 25;
+                        }
+                }
+
+                /* Look for a special ID */
+                if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
+                {
+                        if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
+                                $weight -= 25;
+                        }
+                        if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
+                                $weight += 25;
+                        }
+                }
+                return $weight;
+        }
+
+        /**
+        * Remove extraneous break tags from a node.
+        *
+        * @param DOMElement $node
+        * @return void
+        */
+        public function killBreaks($node) {
+                $html = $node->innerHTML;
+                $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
+                $node->innerHTML = $html;
+        }
+
+        /**
+        * Clean a node of all elements of type "tag".
+        * (Unless it's a youtube/vimeo video. People love movies.)
+        *
+        * Updated 2012-09-18 to preserve youtube/vimeo iframes
+        *
+        * @param DOMElement $e
+        * @param string $tag
+        * @return void
+        */
+        public function clean($e, $tag) {
+                $targetList = $e->getElementsByTagName($tag);
+                $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
+                
+                for ($y=$targetList->length-1; $y >= 0; $y--) {
+                        /* Allow youtube and vimeo videos through as people usually want to see those. */
+                        if ($isEmbed) {
+                                $attributeValues = '';
+                                for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
+                                        $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
+                                }
+                                
+                                /* First, check the elements attributes to see if any of them contain youtube or vimeo */
+                                if (preg_match($this->regexps['video'], $attributeValues)) {
+                                        continue;
+                                }
+
+                                /* Then check the elements inside this element for the same. */
+                                if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
+                                        continue;
+                                }
+                        }
+                        $targetList->item($y)->parentNode->removeChild($targetList->item($y));
+                }
+        }
+        
+        /**
+        * Clean an element of all tags of type "tag" if they look fishy.
+        * "Fishy" is an algorithm based on content length, classnames, 
+        * link density, number of images & embeds, etc.
+        *
+        * @param DOMElement $e
+        * @param string $tag
+        * @return void
+        */
+        public function cleanConditionally($e, $tag) {
+                if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
+                        return;
+                }
+
+                $tagsList = $e->getElementsByTagName($tag);
+                $curTagsLength = $tagsList->length;
+
+                /**
+                * Gather counts for other typical elements embedded within.
+                * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
+                *
+                * TODO: Consider taking into account original contentScore here.
+                */
+                for ($i=$curTagsLength-1; $i >= 0; $i--) {
+                        $weight = $this->getClassWeight($tagsList->item($i));
+                        $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
+                        
+                        $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
+
+                        if ($weight + $contentScore < 0) {
+                                $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
+                        }
+                        else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
+                                /**
+                                * If there are not very many commas, and the number of
+                                * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
+                                **/
+                                $p      = $tagsList->item($i)->getElementsByTagName('p')->length;
+                                $img    = $tagsList->item($i)->getElementsByTagName('img')->length;
+                                $li     = $tagsList->item($i)->getElementsByTagName('li')->length-100;
+                                $input  = $tagsList->item($i)->getElementsByTagName('input')->length;
+                                $a              = $tagsList->item($i)->getElementsByTagName('a')->length;
+
+                                $embedCount = 0;
+                                $embeds = $tagsList->item($i)->getElementsByTagName('embed');
+                                for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
+                                        if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
+                                                $embedCount++; 
+                                        }
+                                }
+                                $embeds = $tagsList->item($i)->getElementsByTagName('iframe');
+                                for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
+                                        if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
+                                                $embedCount++; 
+                                        }
+                                }
+
+                                $linkDensity   = $this->getLinkDensity($tagsList->item($i));
+                                $contentLength = strlen($this->getInnerText($tagsList->item($i)));
+                                $toRemove      = false;
+
+                                if ($this->lightClean) {
+                                        $this->dbg('Light clean...');
+                                        if ( ($img > $p) && ($img > 4) ) {
+                                                $this->dbg(' more than 4 images and more image elements than paragraph elements');
+                                                $toRemove = true;
+                                        } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+                                                $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
+                                                $toRemove = true;
+                                        } else if ( $input > floor($p/3) ) {
+                                                $this->dbg(' too many <input> elements');
+                                                $toRemove = true; 
+                                        } else if ($contentLength < 10 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
+                                                $this->dbg(' content length less than 10 chars, 0 embeds and either 0 images or more than 2 images');
+                                                $toRemove = true;
+                                        } else if($weight < 25 && $linkDensity > 0.2) {
+                                                $this->dbg(' weight smaller than 25 and link density above 0.2');
+                                                $toRemove = true;
+                                        } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
+                                                $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
+                                                $toRemove = true;
+                                        } else if($embedCount > 3) {
+                                                $this->dbg(' more than 3 embeds');
+                                                $toRemove = true;
+                                        }
+                                } else {
+                                        $this->dbg('Standard clean...');
+                                        if ( $img > $p ) {
+                                                $this->dbg(' more image elements than paragraph elements');
+                                                $toRemove = true;
+                                        } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+                                                $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
+                                                $toRemove = true;
+                                        } else if ( $input > floor($p/3) ) {
+                                                $this->dbg(' too many <input> elements');
+                                                $toRemove = true; 
+                                        } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
+                                                $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
+                                                $toRemove = true;
+                                        } else if($weight < 25 && $linkDensity > 0.2) {
+                                                $this->dbg(' weight smaller than 25 and link density above 0.2');
+                                                $toRemove = true;
+                                        } else if($weight >= 25 && $linkDensity > 0.5) {
+                                                $this->dbg(' weight above 25 but link density greater than 0.5');
+                                                $toRemove = true;
+                                        } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
+                                                $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
+                                                $toRemove = true;
+                                        }
+                                }
+
+                                if ($toRemove) {
+                                        //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
+                                        $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
+                                }
+                        }
+                }
+        }
+
+        /**
+        * Clean out spurious headers from an Element. Checks things like classnames and link density.
+        *
+        * @param DOMElement $e
+        * @return void
+        */
+        public function cleanHeaders($e) {
+                for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
+                        $headers = $e->getElementsByTagName('h' . $headerIndex);
+                        for ($i=$headers->length-1; $i >=0; $i--) {
+                                if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
+                                        $headers->item($i)->parentNode->removeChild($headers->item($i));
+                                }
+                        }
+                }
+        }
+
+        public function flagIsActive($flag) {
+                return ($this->flags & $flag) > 0;
+        }
+        
+        public function addFlag($flag) {
+                $this->flags = $this->flags | $flag;
+        }
+        
+        public function removeFlag($flag) {
+                $this->flags = $this->flags & ~$flag;
+        }
+}
 ?>
\ No newline at end of file