-<?php\r
-/** \r
-* Arc90's Readability ported to PHP for FiveFilters.org\r
-* Based on readability.js version 1.7.1 (without multi-page support)\r
-* Updated to allow HTML5 parsing with html5lib\r
-* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds\r
-* ------------------------------------------------------\r
-* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js\r
-* Arc90's project URL: http://lab.arc90.com/experiments/readability/\r
-* JS Source: http://code.google.com/p/arc90labs-readability\r
-* Ported by: Keyvan Minoukadeh, http://www.keyvan.net\r
-* More information: http://fivefilters.org/content-only/\r
-* License: Apache License, Version 2.0\r
-* Requires: PHP5\r
-* Date: 2012-09-19\r
-* \r
-* Differences between the PHP port and the original\r
-* ------------------------------------------------------\r
-* Arc90's Readability is designed to run in the browser. It works on the DOM \r
-* tree (the parsed HTML) after the page's CSS styles have been applied and \r
-* Javascript code executed. This PHP port does not run inside a browser. \r
-* We use PHP's ability to parse HTML to build our DOM tree, but we cannot \r
-* rely on CSS or Javascript support. As such, the results will not always \r
-* match Arc90's Readability. (For example, if a web page contains CSS style \r
-* rules or Javascript code which hide certain HTML elements from display, \r
-* Arc90's Readability will dismiss those from consideration but our PHP port, \r
-* unable to understand CSS or Javascript, will not know any better.)\r
-* \r
-* Another significant difference is that the aim of Arc90's Readability is \r
-* to re-present the main content block of a given web page so users can \r
-* read it more easily in their browsers. Correct identification, clean up, \r
-* and separation of the content block is only a part of this process. \r
-* This PHP port is only concerned with this part, it does not include code \r
-* that relates to presentation in the browser - Arc90 already do \r
-* that extremely well, and for PDF output there's FiveFilters.org's \r
-* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.\r
-* \r
-* Finally, this class contains methods that might be useful for developers \r
-* working on HTML document fragments. So without deviating too much from \r
-* the original code (which I don't want to do because it makes debugging \r
-* and updating more difficult), I've tried to make it a little more \r
-* developer friendly. You should be able to use the methods here on \r
-* existing DOMElement objects without passing an entire HTML document to \r
-* be parsed.\r
-*/\r
-\r
-// This class allows us to do JavaScript like assignements to innerHTML\r
-require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');\r
-\r
-// Alternative usage (for testing only!)\r
-// uncomment the lines below and call Readability.php in your browser \r
-// passing it the URL of the page you'd like content from, e.g.:\r
-// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php\r
-\r
-/*\r
-if (!isset($_GET['url']) || $_GET['url'] == '') {\r
- die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');\r
-}\r
-$url = $_GET['url'];\r
-if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;\r
-$html = file_get_contents($url);\r
-$r = new Readability($html, $url);\r
-$r->init();\r
-echo $r->articleContent->innerHTML;\r
-*/\r
-\r
-class Readability\r
-{\r
- public $version = '1.7.1-without-multi-page';\r
- public $convertLinksToFootnotes = false;\r
- public $revertForcedParagraphElements = true;\r
- public $articleTitle;\r
- public $articleContent;\r
- public $dom;\r
- public $url = null; // optional - URL where HTML was retrieved\r
- public $debug = false;\r
- public $lightClean = true; // preserves more content (experimental) added 2012-09-19\r
- protected $body = null; // \r
- protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later\r
- protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.\r
- protected $success = false; // indicates whether we were able to extract or not\r
- \r
- /**\r
- * All of the regular expressions in use within readability.\r
- * Defined up here so we don't instantiate them repeatedly in loops.\r
- **/\r
- public $regexps = array(\r
- 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',\r
- 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',\r
- 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',\r
- 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',\r
- 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',\r
- 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',\r
- 'replaceFonts' => '/<(\/?)font[^>]*>/i',\r
- // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()\r
- 'normalize' => '/\s{2,}/',\r
- 'killBreaks' => '/(<br\s*\/?>(\s| ?)*){1,}/',\r
- 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',\r
- 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'\r
- ); \r
- \r
- /* constants */\r
- const FLAG_STRIP_UNLIKELYS = 1;\r
- const FLAG_WEIGHT_CLASSES = 2;\r
- const FLAG_CLEAN_CONDITIONALLY = 4;\r
- \r
- /**\r
- * Create instance of Readability\r
- * @param string UTF-8 encoded string\r
- * @param string (optional) URL associated with HTML (used for footnotes)\r
- * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')\r
- */ \r
- function __construct($html, $url=null, $parser='libxml')\r
- {\r
- $this->url = $url;\r
- /* Turn all double br's into p's */\r
- $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);\r
- $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);\r
- $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");\r
- if (trim($html) == '') $html = '<html></html>';\r
- if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {\r
- // all good\r
- } else {\r
- $this->dom = new DOMDocument();\r
- $this->dom->preserveWhiteSpace = false;\r
- @$this->dom->loadHTML($html);\r
- }\r
- $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');\r
- }\r
-\r
- /**\r
- * Get article title element\r
- * @return DOMElement\r
- */\r
- public function getTitle() {\r
- return $this->articleTitle;\r
- }\r
- \r
- /**\r
- * Get article content element\r
- * @return DOMElement\r
- */\r
- public function getContent() {\r
- return $this->articleContent;\r
- } \r
- \r
- /**\r
- * Runs readability.\r
- * \r
- * Workflow:\r
- * 1. Prep the document by removing script tags, css, etc.\r
- * 2. Build readability's DOM tree.\r
- * 3. Grab the article content from the current dom tree.\r
- * 4. Replace the current DOM tree with the new one.\r
- * 5. Read peacefully.\r
- *\r
- * @return boolean true if we found content, false otherwise\r
- **/\r
- public function init()\r
- {\r
- if (!isset($this->dom->documentElement)) return false;\r
- $this->removeScripts($this->dom);\r
- //die($this->getInnerHTML($this->dom->documentElement));\r
- \r
- // Assume successful outcome\r
- $this->success = true;\r
-\r
- $bodyElems = $this->dom->getElementsByTagName('body');\r
- if ($bodyElems->length > 0) {\r
- if ($this->bodyCache == null) {\r
- $this->bodyCache = $bodyElems->item(0)->innerHTML;\r
- }\r
- if ($this->body == null) {\r
- $this->body = $bodyElems->item(0);\r
- }\r
- }\r
-\r
- $this->prepDocument();\r
- \r
- //die($this->dom->documentElement->parentNode->nodeType);\r
- //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));\r
- //die($this->getInnerHTML($this->dom->documentElement));\r
-\r
- /* Build readability's DOM tree */\r
- $overlay = $this->dom->createElement('div');\r
- $innerDiv = $this->dom->createElement('div');\r
- $articleTitle = $this->getArticleTitle();\r
- $articleContent = $this->grabArticle();\r
-\r
- if (!$articleContent) {\r
- $this->success = false;\r
- $articleContent = $this->dom->createElement('div');\r
- $articleContent->setAttribute('id', 'readability-content');\r
- $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>'; \r
- }\r
- \r
- $overlay->setAttribute('id', 'readOverlay');\r
- $innerDiv->setAttribute('id', 'readInner');\r
-\r
- /* Glue the structure of our document together. */\r
- $innerDiv->appendChild($articleTitle);\r
- $innerDiv->appendChild($articleContent);\r
- $overlay->appendChild($innerDiv);\r
- \r
- /* Clear the old HTML, insert the new content. */\r
- $this->body->innerHTML = '';\r
- $this->body->appendChild($overlay);\r
- //document.body.insertBefore(overlay, document.body.firstChild);\r
- $this->body->removeAttribute('style');\r
-\r
- $this->postProcessContent($articleContent);\r
- \r
- // Set title and content instance variables\r
- $this->articleTitle = $articleTitle;\r
- $this->articleContent = $articleContent;\r
- \r
- return $this->success;\r
- }\r
- \r
- /**\r
- * Debug\r
- */\r
- protected function dbg($msg) {\r
- if ($this->debug) echo '* ',$msg, "\n";\r
- }\r
- \r
- /**\r
- * Run any post-process modifications to article content as necessary.\r
- *\r
- * @param DOMElement\r
- * @return void\r
- */\r
- public function postProcessContent($articleContent) {\r
- if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { \r
- $this->addFootnotes($articleContent);\r
- }\r
- }\r
- \r
- /**\r
- * Get the article title as an H1.\r
- *\r
- * @return DOMElement\r
- */\r
- protected function getArticleTitle() {\r
- $curTitle = '';\r
- $origTitle = '';\r
-\r
- try {\r
- $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));\r
- } catch(Exception $e) {}\r
- \r
- if (preg_match('/ [\|\-] /', $curTitle))\r
- {\r
- $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);\r
- \r
- if (count(explode(' ', $curTitle)) < 3) {\r
- $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);\r
- }\r
- }\r
- else if (strpos($curTitle, ': ') !== false)\r
- {\r
- $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);\r
-\r
- if (count(explode(' ', $curTitle)) < 3) {\r
- $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);\r
- }\r
- }\r
- else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)\r
- {\r
- $hOnes = $this->dom->getElementsByTagName('h1');\r
- if($hOnes->length == 1)\r
- {\r
- $curTitle = $this->getInnerText($hOnes->item(0));\r
- }\r
- }\r
-\r
- $curTitle = trim($curTitle);\r
-\r
- if (count(explode(' ', $curTitle)) <= 4) {\r
- $curTitle = $origTitle;\r
- }\r
- \r
- $articleTitle = $this->dom->createElement('h1');\r
- $articleTitle->innerHTML = $curTitle;\r
- \r
- return $articleTitle;\r
- }\r
- \r
- /**\r
- * Prepare the HTML document for readability to scrape it.\r
- * This includes things like stripping javascript, CSS, and handling terrible markup.\r
- * \r
- * @return void\r
- **/\r
- protected function prepDocument() {\r
- /**\r
- * In some cases a body element can't be found (if the HTML is totally hosed for example)\r
- * so we create a new body node and append it to the document.\r
- */\r
- if ($this->body == null)\r
- {\r
- $this->body = $this->dom->createElement('body');\r
- $this->dom->documentElement->appendChild($this->body);\r
- }\r
- $this->body->setAttribute('id', 'readabilityBody');\r
-\r
- /* Remove all style tags in head */\r
- $styleTags = $this->dom->getElementsByTagName('style');\r
- for ($i = $styleTags->length-1; $i >= 0; $i--)\r
- {\r
- $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));\r
- }\r
-\r
- /* Turn all double br's into p's */\r
- /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */\r
- //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');\r
- // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.\r
- // Manipulating innerHTML as it's done in JS is not possible in PHP.\r
- }\r
-\r
- /**\r
- * For easier reading, convert this document to have footnotes at the bottom rather than inline links.\r
- * @see http://www.roughtype.com/archives/2010/05/experiments_in.php\r
- *\r
- * @return void\r
- **/\r
- public function addFootnotes($articleContent) {\r
- $footnotesWrapper = $this->dom->createElement('div');\r
- $footnotesWrapper->setAttribute('id', 'readability-footnotes');\r
- $footnotesWrapper->innerHTML = '<h3>References</h3>';\r
- \r
- $articleFootnotes = $this->dom->createElement('ol');\r
- $articleFootnotes->setAttribute('id', 'readability-footnotes-list');\r
- $footnotesWrapper->appendChild($articleFootnotes);\r
- \r
- $articleLinks = $articleContent->getElementsByTagName('a');\r
- \r
- $linkCount = 0;\r
- for ($i = 0; $i < $articleLinks->length; $i++)\r
- {\r
- $articleLink = $articleLinks->item($i);\r
- $footnoteLink = $articleLink->cloneNode(true);\r
- $refLink = $this->dom->createElement('a');\r
- $footnote = $this->dom->createElement('li');\r
- $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);\r
- if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);\r
- //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,\r
- $linkText = $this->getInnerText($articleLink);\r
- \r
- if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {\r
- continue;\r
- }\r
- \r
- $linkCount++;\r
-\r
- /** Add a superscript reference after the article link */\r
- $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);\r
- $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';\r
- $refLink->setAttribute('class', 'readability-DoNotFootnote');\r
- $refLink->setAttribute('style', 'color: inherit;');\r
- \r
- //TODO: does this work or should we use DOMNode.isSameNode()?\r
- if ($articleLink->parentNode->lastChild == $articleLink) {\r
- $articleLink->parentNode->appendChild($refLink);\r
- } else {\r
- $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);\r
- }\r
-\r
- $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');\r
- $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);\r
-\r
- $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';\r
-\r
- $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);\r
- $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);\r
- \r
- $footnote->appendChild($footnoteLink);\r
- if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';\r
- \r
- $articleFootnotes->appendChild($footnote);\r
- }\r
-\r
- if ($linkCount > 0) {\r
- $articleContent->appendChild($footnotesWrapper); \r
- }\r
- }\r
-\r
- /**\r
- * Reverts P elements with class 'readability-styled'\r
- * to text nodes - which is what they were before.\r
- *\r
- * @param DOMElement\r
- * @return void\r
- */\r
- function revertReadabilityStyledElements($articleContent) {\r
- $xpath = new DOMXPath($articleContent->ownerDocument);\r
- $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);\r
- //$elems = $articleContent->getElementsByTagName('p');\r
- for ($i = $elems->length-1; $i >= 0; $i--) {\r
- $e = $elems->item($i);\r
- $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);\r
- //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {\r
- // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);\r
- //}\r
- }\r
- }\r
- \r
- /**\r
- * Prepare the article node for display. Clean out any inline styles,\r
- * iframes, forms, strip extraneous <p> tags, etc.\r
- *\r
- * @param DOMElement\r
- * @return void\r
- */\r
- function prepArticle($articleContent) {\r
- $this->cleanStyles($articleContent);\r
- $this->killBreaks($articleContent);\r
- if ($this->revertForcedParagraphElements) {\r
- $this->revertReadabilityStyledElements($articleContent);\r
- }\r
-\r
- /* Clean out junk from the article content */\r
- $this->cleanConditionally($articleContent, 'form');\r
- $this->clean($articleContent, 'object');\r
- $this->clean($articleContent, 'h1');\r
-\r
- /**\r
- * If there is only one h2, they are probably using it\r
- * as a header and not a subheader, so remove it since we already have a header.\r
- ***/\r
- if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {\r
- $this->clean($articleContent, 'h2'); \r
- }\r
- $this->clean($articleContent, 'iframe');\r
-\r
- $this->cleanHeaders($articleContent);\r
-\r
- /* Do these last as the previous stuff may have removed junk that will affect these */\r
- $this->cleanConditionally($articleContent, 'table');\r
- $this->cleanConditionally($articleContent, 'ul');\r
- $this->cleanConditionally($articleContent, 'div');\r
-\r
- /* Remove extra paragraphs */\r
- $articleParagraphs = $articleContent->getElementsByTagName('p');\r
- for ($i = $articleParagraphs->length-1; $i >= 0; $i--)\r
- {\r
- $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;\r
- $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;\r
- $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;\r
- $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;\r
- \r
- if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')\r
- {\r
- $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));\r
- }\r
- }\r
-\r
- try {\r
- $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);\r
- //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p'); \r
- }\r
- catch (Exception $e) {\r
- $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);\r
- }\r
- }\r
- \r
- /**\r
- * Initialize a node with the readability object. Also checks the\r
- * className/id for special names to add to its score.\r
- *\r
- * @param Element\r
- * @return void\r
- **/\r
- protected function initializeNode($node) {\r
- $readability = $this->dom->createAttribute('readability');\r
- $readability->value = 0; // this is our contentScore\r
- $node->setAttributeNode($readability); \r
-\r
- switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case\r
- case 'DIV':\r
- $readability->value += 5;\r
- break;\r
-\r
- case 'PRE':\r
- case 'TD':\r
- case 'BLOCKQUOTE':\r
- $readability->value += 3;\r
- break;\r
- \r
- case 'ADDRESS':\r
- case 'OL':\r
- case 'UL':\r
- case 'DL':\r
- case 'DD':\r
- case 'DT':\r
- case 'LI':\r
- case 'FORM':\r
- $readability->value -= 3;\r
- break;\r
-\r
- case 'H1':\r
- case 'H2':\r
- case 'H3':\r
- case 'H4':\r
- case 'H5':\r
- case 'H6':\r
- case 'TH':\r
- $readability->value -= 5;\r
- break;\r
- }\r
- $readability->value += $this->getClassWeight($node);\r
- }\r
- \r
- /***\r
- * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is\r
- * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.\r
- *\r
- * @return DOMElement\r
- **/\r
- protected function grabArticle($page=null) {\r
- $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);\r
- if (!$page) $page = $this->dom;\r
- $allElements = $page->getElementsByTagName('*');\r
- /**\r
- * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs\r
- * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)\r
- *\r
- * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5\r
- * TODO: Shouldn't this be a reverse traversal?\r
- **/\r
- $node = null;\r
- $nodesToScore = array();\r
- for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {\r
- //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {\r
- //$node = $targetList->item($nodeIndex);\r
- $tagName = strtoupper($node->tagName);\r
- /* Remove unlikely candidates */\r
- if ($stripUnlikelyCandidates) {\r
- $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');\r
- if (\r
- preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&\r
- !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&\r
- $tagName != 'BODY'\r
- )\r
- {\r
- $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);\r
- //$nodesToRemove[] = $node;\r
- $node->parentNode->removeChild($node);\r
- $nodeIndex--;\r
- continue;\r
- } \r
- }\r
-\r
- if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {\r
- $nodesToScore[] = $node;\r
- }\r
-\r
- /* Turn all divs that don't have children block level elements into p's */\r
- if ($tagName == 'DIV') {\r
- if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {\r
- //$this->dbg('Altering div to p');\r
- $newNode = $this->dom->createElement('p');\r
- try {\r
- $newNode->innerHTML = $node->innerHTML;\r
- //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);\r
- $node->parentNode->replaceChild($newNode, $node);\r
- $nodeIndex--;\r
- $nodesToScore[] = $node; // or $newNode?\r
- }\r
- catch(Exception $e) {\r
- $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);\r
- }\r
- }\r
- else\r
- {\r
- /* EXPERIMENTAL */\r
- // TODO: change these p elements back to text nodes after processing\r
- for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {\r
- $childNode = $node->childNodes->item($i);\r
- if ($childNode->nodeType == 3) { // XML_TEXT_NODE\r
- //$this->dbg('replacing text node with a p tag with the same content.');\r
- $p = $this->dom->createElement('p');\r
- $p->innerHTML = $childNode->nodeValue;\r
- $p->setAttribute('style', 'display: inline;');\r
- $p->setAttribute('class', 'readability-styled');\r
- $childNode->parentNode->replaceChild($p, $childNode);\r
- }\r
- }\r
- }\r
- }\r
- }\r
- \r
- /**\r
- * Loop through all paragraphs, and assign a score to them based on how content-y they look.\r
- * Then add their score to their parent node.\r
- *\r
- * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.\r
- **/\r
- $candidates = array();\r
- for ($pt=0; $pt < count($nodesToScore); $pt++) {\r
- $parentNode = $nodesToScore[$pt]->parentNode;\r
- // $grandParentNode = $parentNode ? $parentNode->parentNode : null;\r
- $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);\r
- $innerText = $this->getInnerText($nodesToScore[$pt]);\r
-\r
- if (!$parentNode || !isset($parentNode->tagName)) {\r
- continue;\r
- }\r
-\r
- /* If this paragraph is less than 25 characters, don't even count it. */\r
- if(strlen($innerText) < 25) {\r
- continue;\r
- }\r
-\r
- /* Initialize readability data for the parent. */\r
- if (!$parentNode->hasAttribute('readability')) \r
- {\r
- $this->initializeNode($parentNode);\r
- $candidates[] = $parentNode;\r
- }\r
-\r
- /* Initialize readability data for the grandparent. */\r
- if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))\r
- {\r
- $this->initializeNode($grandParentNode);\r
- $candidates[] = $grandParentNode;\r
- }\r
-\r
- $contentScore = 0;\r
-\r
- /* Add a point for the paragraph itself as a base. */\r
- $contentScore++;\r
-\r
- /* Add points for any commas within this paragraph */\r
- $contentScore += count(explode(',', $innerText));\r
- \r
- /* For every 100 characters in this paragraph, add another point. Up to 3 points. */\r
- $contentScore += min(floor(strlen($innerText) / 100), 3);\r
- \r
- /* Add the score to the parent. The grandparent gets half. */\r
- $parentNode->getAttributeNode('readability')->value += $contentScore;\r
-\r
- if ($grandParentNode) {\r
- $grandParentNode->getAttributeNode('readability')->value += $contentScore/2; \r
- }\r
- }\r
-\r
- /**\r
- * After we've calculated scores, loop through all of the possible candidate nodes we found\r
- * and find the one with the highest score.\r
- **/\r
- $topCandidate = null;\r
- for ($c=0, $cl=count($candidates); $c < $cl; $c++)\r
- {\r
- /**\r
- * Scale the final candidates score based on link density. Good content should have a\r
- * relatively small link density (5% or less) and be mostly unaffected by this operation.\r
- **/\r
- $readability = $candidates[$c]->getAttributeNode('readability');\r
- $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));\r
-\r
- $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);\r
-\r
- if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {\r
- $topCandidate = $candidates[$c];\r
- }\r
- }\r
-\r
- /**\r
- * If we still have no top candidate, just use the body as a last resort.\r
- * We also have to copy the body node so it is something we can modify.\r
- **/\r
- if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')\r
- {\r
- $topCandidate = $this->dom->createElement('div');\r
- if ($page instanceof DOMDocument) {\r
- if (!isset($page->documentElement)) {\r
- // we don't have a body either? what a mess! :)\r
- } else {\r
- $topCandidate->innerHTML = $page->documentElement->innerHTML;\r
- $page->documentElement->innerHTML = '';\r
- $page->documentElement->appendChild($topCandidate);\r
- }\r
- } else {\r
- $topCandidate->innerHTML = $page->innerHTML;\r
- $page->innerHTML = '';\r
- $page->appendChild($topCandidate);\r
- }\r
- $this->initializeNode($topCandidate);\r
- }\r
-\r
- /**\r
- * Now that we have the top candidate, look through its siblings for content that might also be related.\r
- * Things like preambles, content split by ads that we removed, etc.\r
- **/\r
- $articleContent = $this->dom->createElement('div');\r
- $articleContent->setAttribute('id', 'readability-content');\r
- $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);\r
- $siblingNodes = $topCandidate->parentNode->childNodes;\r
- if (!isset($siblingNodes)) {\r
- $siblingNodes = new stdClass;\r
- $siblingNodes->length = 0;\r
- }\r
-\r
- for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)\r
- {\r
- $siblingNode = $siblingNodes->item($s);\r
- $append = false;\r
-\r
- $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));\r
-\r
- //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));\r
-\r
- if ($siblingNode === $topCandidate)\r
- // or if ($siblingNode->isSameNode($topCandidate))\r
- {\r
- $append = true;\r
- }\r
-\r
- $contentBonus = 0;\r
- /* Give a bonus if sibling nodes and top candidates have the example same classname */\r
- if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {\r
- $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;\r
- }\r
-\r
- if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)\r
- {\r
- $append = true;\r
- }\r
- \r
- if (strtoupper($siblingNode->nodeName) == 'P') {\r
- $linkDensity = $this->getLinkDensity($siblingNode);\r
- $nodeContent = $this->getInnerText($siblingNode);\r
- $nodeLength = strlen($nodeContent);\r
- \r
- if ($nodeLength > 80 && $linkDensity < 0.25)\r
- {\r
- $append = true;\r
- }\r
- else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))\r
- {\r
- $append = true;\r
- }\r
- }\r
-\r
- if ($append)\r
- {\r
- $this->dbg('Appending node: ' . $siblingNode->nodeName);\r
-\r
- $nodeToAppend = null;\r
- $sibNodeName = strtoupper($siblingNode->nodeName);\r
- if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {\r
- /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */\r
- \r
- $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');\r
- $nodeToAppend = $this->dom->createElement('div');\r
- try {\r
- $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));\r
- $nodeToAppend->innerHTML = $siblingNode->innerHTML;\r
- }\r
- catch(Exception $e)\r
- {\r
- $this->dbg('Could not alter siblingNode to div, reverting back to original.');\r
- $nodeToAppend = $siblingNode;\r
- $s--;\r
- $sl--;\r
- }\r
- } else {\r
- $nodeToAppend = $siblingNode;\r
- $s--;\r
- $sl--;\r
- }\r
- \r
- /* To ensure a node does not interfere with readability styles, remove its classnames */\r
- $nodeToAppend->removeAttribute('class');\r
-\r
- /* Append sibling and subtract from our list because it removes the node when you append to another node */\r
- $articleContent->appendChild($nodeToAppend);\r
- }\r
- }\r
-\r
- /**\r
- * So we have all of the content that we need. Now we clean it up for presentation.\r
- **/\r
- $this->prepArticle($articleContent);\r
-\r
- /**\r
- * Now that we've gone through the full algorithm, check to see if we got any meaningful content.\r
- * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher\r
- * likelihood of finding the content, and the sieve approach gives us a higher likelihood of\r
- * finding the -right- content.\r
- **/\r
- if (strlen($this->getInnerText($articleContent, false)) < 250)\r
- {\r
- // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7\r
- // in the meantime, we check and create an empty element if it's not there.\r
- if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');\r
- $this->body->innerHTML = $this->bodyCache;\r
- \r
- if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {\r
- $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);\r
- return $this->grabArticle($this->body);\r
- }\r
- else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {\r
- $this->removeFlag(self::FLAG_WEIGHT_CLASSES);\r
- return $this->grabArticle($this->body); \r
- }\r
- else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {\r
- $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);\r
- return $this->grabArticle($this->body);\r
- }\r
- else {\r
- return false;\r
- }\r
- }\r
- return $articleContent;\r
- }\r
- \r
- /**\r
- * Remove script tags from document\r
- *\r
- * @param DOMElement\r
- * @return void\r
- */\r
- public function removeScripts($doc) {\r
- $scripts = $doc->getElementsByTagName('script');\r
- for($i = $scripts->length-1; $i >= 0; $i--)\r
- {\r
- $scripts->item($i)->parentNode->removeChild($scripts->item($i));\r
- }\r
- }\r
- \r
- /**\r
- * Get the inner text of a node.\r
- * This also strips out any excess whitespace to be found.\r
- *\r
- * @param DOMElement $\r
- * @param boolean $normalizeSpaces (default: true)\r
- * @return string\r
- **/\r
- public function getInnerText($e, $normalizeSpaces=true) {\r
- $textContent = '';\r
-\r
- if (!isset($e->textContent) || $e->textContent == '') {\r
- return '';\r
- }\r
-\r
- $textContent = trim($e->textContent);\r
-\r
- if ($normalizeSpaces) {\r
- return preg_replace($this->regexps['normalize'], ' ', $textContent);\r
- } else {\r
- return $textContent;\r
- }\r
- }\r
-\r
- /**\r
- * Get the number of times a string $s appears in the node $e.\r
- *\r
- * @param DOMElement $e\r
- * @param string - what to count. Default is ","\r
- * @return number (integer)\r
- **/\r
- public function getCharCount($e, $s=',') {\r
- return substr_count($this->getInnerText($e), $s);\r
- }\r
-\r
- /**\r
- * Remove the style attribute on every $e and under.\r
- *\r
- * @param DOMElement $e\r
- * @return void\r
- */\r
- public function cleanStyles($e) {\r
- if (!is_object($e)) return;\r
- $elems = $e->getElementsByTagName('*');\r
- foreach ($elems as $elem) {\r
- $elem->removeAttribute('style');\r
- }\r
- }\r
- \r
- /**\r
- * Get the density of links as a percentage of the content\r
- * This is the amount of text that is inside a link divided by the total text in the node.\r
- * \r
- * @param DOMElement $e\r
- * @return number (float)\r
- */\r
- public function getLinkDensity($e) {\r
- $links = $e->getElementsByTagName('a');\r
- $textLength = strlen($this->getInnerText($e));\r
- $linkLength = 0;\r
- for ($i=0, $il=$links->length; $i < $il; $i++)\r
- {\r
- $linkLength += strlen($this->getInnerText($links->item($i)));\r
- }\r
- if ($textLength > 0) {\r
- return $linkLength / $textLength;\r
- } else {\r
- return 0;\r
- }\r
- }\r
- \r
- /**\r
- * Get an elements class/id weight. Uses regular expressions to tell if this \r
- * element looks good or bad.\r
- *\r
- * @param DOMElement $e\r
- * @return number (Integer)\r
- */\r
- public function getClassWeight($e) {\r
- if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {\r
- return 0;\r
- }\r
-\r
- $weight = 0;\r
-\r
- /* Look for a special classname */\r
- if ($e->hasAttribute('class') && $e->getAttribute('class') != '')\r
- {\r
- if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {\r
- $weight -= 25;\r
- }\r
- if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {\r
- $weight += 25;\r
- }\r
- }\r
-\r
- /* Look for a special ID */\r
- if ($e->hasAttribute('id') && $e->getAttribute('id') != '')\r
- {\r
- if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {\r
- $weight -= 25;\r
- }\r
- if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {\r
- $weight += 25;\r
- }\r
- }\r
- return $weight;\r
- }\r
-\r
- /**\r
- * Remove extraneous break tags from a node.\r
- *\r
- * @param DOMElement $node\r
- * @return void\r
- */\r
- public function killBreaks($node) {\r
- $html = $node->innerHTML;\r
- $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);\r
- $node->innerHTML = $html;\r
- }\r
-\r
- /**\r
- * Clean a node of all elements of type "tag".\r
- * (Unless it's a youtube/vimeo video. People love movies.)\r
- *\r
- * Updated 2012-09-18 to preserve youtube/vimeo iframes\r
- *\r
- * @param DOMElement $e\r
- * @param string $tag\r
- * @return void\r
- */\r
- public function clean($e, $tag) {\r
- $targetList = $e->getElementsByTagName($tag);\r
- $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');\r
- \r
- for ($y=$targetList->length-1; $y >= 0; $y--) {\r
- /* Allow youtube and vimeo videos through as people usually want to see those. */\r
- if ($isEmbed) {\r
- $attributeValues = '';\r
- for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {\r
- $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)\r
- }\r
- \r
- /* First, check the elements attributes to see if any of them contain youtube or vimeo */\r
- if (preg_match($this->regexps['video'], $attributeValues)) {\r
- continue;\r
- }\r
-\r
- /* Then check the elements inside this element for the same. */\r
- if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {\r
- continue;\r
- }\r
- }\r
- $targetList->item($y)->parentNode->removeChild($targetList->item($y));\r
- }\r
- }\r
- \r
- /**\r
- * Clean an element of all tags of type "tag" if they look fishy.\r
- * "Fishy" is an algorithm based on content length, classnames, \r
- * link density, number of images & embeds, etc.\r
- *\r
- * @param DOMElement $e\r
- * @param string $tag\r
- * @return void\r
- */\r
- public function cleanConditionally($e, $tag) {\r
- if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {\r
- return;\r
- }\r
-\r
- $tagsList = $e->getElementsByTagName($tag);\r
- $curTagsLength = $tagsList->length;\r
-\r
- /**\r
- * Gather counts for other typical elements embedded within.\r
- * Traverse backwards so we can remove nodes at the same time without effecting the traversal.\r
- *\r
- * TODO: Consider taking into account original contentScore here.\r
- */\r
- for ($i=$curTagsLength-1; $i >= 0; $i--) {\r
- $weight = $this->getClassWeight($tagsList->item($i));\r
- $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;\r
- \r
- $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));\r
-\r
- if ($weight + $contentScore < 0) {\r
- $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));\r
- }\r
- else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {\r
- /**\r
- * If there are not very many commas, and the number of\r
- * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.\r
- **/\r
- $p = $tagsList->item($i)->getElementsByTagName('p')->length;\r
- $img = $tagsList->item($i)->getElementsByTagName('img')->length;\r
- $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;\r
- $input = $tagsList->item($i)->getElementsByTagName('input')->length;\r
- $a = $tagsList->item($i)->getElementsByTagName('a')->length;\r
-\r
- $embedCount = 0;\r
- $embeds = $tagsList->item($i)->getElementsByTagName('embed');\r
- for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {\r
- if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {\r
- $embedCount++; \r
- }\r
- }\r
- $embeds = $tagsList->item($i)->getElementsByTagName('iframe');\r
- for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {\r
- if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {\r
- $embedCount++; \r
- }\r
- }\r
-\r
- $linkDensity = $this->getLinkDensity($tagsList->item($i));\r
- $contentLength = strlen($this->getInnerText($tagsList->item($i)));\r
- $toRemove = false;\r
-\r
- if ($this->lightClean) {\r
- $this->dbg('Light clean...');\r
- if ( ($img > $p) && ($img > 4) ) {\r
- $this->dbg(' more than 4 images and more image elements than paragraph elements');\r
- $toRemove = true;\r
- } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {\r
- $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');\r
- $toRemove = true;\r
- } else if ( $input > floor($p/3) ) {\r
- $this->dbg(' too many <input> elements');\r
- $toRemove = true; \r
- } else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {\r
- $this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');\r
- $toRemove = true;\r
- } else if($weight < 25 && $linkDensity > 0.2) {\r
- $this->dbg(' weight smaller than 25 and link density above 0.2');\r
- $toRemove = true;\r
- } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {\r
- $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');\r
- $toRemove = true;\r
- } else if($embedCount > 3) {\r
- $this->dbg(' more than 3 embeds');\r
- $toRemove = true;\r
- }\r
- } else {\r
- $this->dbg('Standard clean...');\r
- if ( $img > $p ) {\r
- $this->dbg(' more image elements than paragraph elements');\r
- $toRemove = true;\r
- } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {\r
- $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');\r
- $toRemove = true;\r
- } else if ( $input > floor($p/3) ) {\r
- $this->dbg(' too many <input> elements');\r
- $toRemove = true; \r
- } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {\r
- $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');\r
- $toRemove = true;\r
- } else if($weight < 25 && $linkDensity > 0.2) {\r
- $this->dbg(' weight smaller than 25 and link density above 0.2');\r
- $toRemove = true;\r
- } else if($weight >= 25 && $linkDensity > 0.5) {\r
- $this->dbg(' weight above 25 but link density greater than 0.5');\r
- $toRemove = true;\r
- } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {\r
- $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');\r
- $toRemove = true;\r
- }\r
- }\r
-\r
- if ($toRemove) {\r
- //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);\r
- $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));\r
- }\r
- }\r
- }\r
- }\r
-\r
- /**\r
- * Clean out spurious headers from an Element. Checks things like classnames and link density.\r
- *\r
- * @param DOMElement $e\r
- * @return void\r
- */\r
- public function cleanHeaders($e) {\r
- for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {\r
- $headers = $e->getElementsByTagName('h' . $headerIndex);\r
- for ($i=$headers->length-1; $i >=0; $i--) {\r
- if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {\r
- $headers->item($i)->parentNode->removeChild($headers->item($i));\r
- }\r
- }\r
- }\r
- }\r
-\r
- public function flagIsActive($flag) {\r
- return ($this->flags & $flag) > 0;\r
- }\r
- \r
- public function addFlag($flag) {\r
- $this->flags = $this->flags | $flag;\r
- }\r
- \r
- public function removeFlag($flag) {\r
- $this->flags = $this->flags & ~$flag;\r
- }\r
-}\r
+<?php
+/**
+* Arc90's Readability ported to PHP for FiveFilters.org
+* Based on readability.js version 1.7.1 (without multi-page support)
+* Updated to allow HTML5 parsing with html5lib
+* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
+* ------------------------------------------------------
+* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
+* Arc90's project URL: http://lab.arc90.com/experiments/readability/
+* JS Source: http://code.google.com/p/arc90labs-readability
+* Ported by: Keyvan Minoukadeh, http://www.keyvan.net
+* More information: http://fivefilters.org/content-only/
+* License: Apache License, Version 2.0
+* Requires: PHP5
+* Date: 2012-09-19
+*
+* Differences between the PHP port and the original
+* ------------------------------------------------------
+* Arc90's Readability is designed to run in the browser. It works on the DOM
+* tree (the parsed HTML) after the page's CSS styles have been applied and
+* Javascript code executed. This PHP port does not run inside a browser.
+* We use PHP's ability to parse HTML to build our DOM tree, but we cannot
+* rely on CSS or Javascript support. As such, the results will not always
+* match Arc90's Readability. (For example, if a web page contains CSS style
+* rules or Javascript code which hide certain HTML elements from display,
+* Arc90's Readability will dismiss those from consideration but our PHP port,
+* unable to understand CSS or Javascript, will not know any better.)
+*
+* Another significant difference is that the aim of Arc90's Readability is
+* to re-present the main content block of a given web page so users can
+* read it more easily in their browsers. Correct identification, clean up,
+* and separation of the content block is only a part of this process.
+* This PHP port is only concerned with this part, it does not include code
+* that relates to presentation in the browser - Arc90 already do
+* that extremely well, and for PDF output there's FiveFilters.org's
+* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
+*
+* Finally, this class contains methods that might be useful for developers
+* working on HTML document fragments. So without deviating too much from
+* the original code (which I don't want to do because it makes debugging
+* and updating more difficult), I've tried to make it a little more
+* developer friendly. You should be able to use the methods here on
+* existing DOMElement objects without passing an entire HTML document to
+* be parsed.
+*/
+
+// This class allows us to do JavaScript like assignements to innerHTML
+require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
+
+// Alternative usage (for testing only!)
+// uncomment the lines below and call Readability.php in your browser
+// passing it the URL of the page you'd like content from, e.g.:
+// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
+
+/*
+if (!isset($_GET['url']) || $_GET['url'] == '') {
+ die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
+}
+$url = $_GET['url'];
+if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
+$html = file_get_contents($url);
+$r = new Readability($html, $url);
+$r->init();
+echo $r->articleContent->innerHTML;
+*/
+
+class Readability
+{
+ public $version = '1.7.1-without-multi-page';
+ public $convertLinksToFootnotes = false;
+ public $revertForcedParagraphElements = true;
+ public $articleTitle;
+ public $articleContent;
+ public $dom;
+ public $url = null; // optional - URL where HTML was retrieved
+ public $debug = false;
+ public $lightClean = true; // preserves more content (experimental) added 2012-09-19
+ protected $body = null; //
+ protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
+ protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
+ protected $success = false; // indicates whether we were able to extract or not
+
+ /**
+ * All of the regular expressions in use within readability.
+ * Defined up here so we don't instantiate them repeatedly in loops.
+ **/
+ public $regexps = array(
+ 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
+ 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
+ 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
+ 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
+ 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
+ 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
+ 'replaceFonts' => '/<(\/?)font[^>]*>/i',
+ // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
+ 'normalize' => '/\s{2,}/',
+ 'killBreaks' => '/(<br\s*\/?>(\s| ?)*){1,}/',
+ 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
+ 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
+ );
+
+ /* constants */
+ const FLAG_STRIP_UNLIKELYS = 1;
+ const FLAG_WEIGHT_CLASSES = 2;
+ const FLAG_CLEAN_CONDITIONALLY = 4;
+
+ /**
+ * Create instance of Readability
+ * @param string UTF-8 encoded string
+ * @param string (optional) URL associated with HTML (used for footnotes)
+ * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
+ */
+ function __construct($html, $url=null, $parser='libxml')
+ {
+ $this->url = $url;
+ /* Turn all double br's into p's */
+ $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
+ $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
+ $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
+ if (trim($html) == '') $html = '<html></html>';
+ if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
+ // all good
+ } else {
+ $this->dom = new DOMDocument();
+ $this->dom->preserveWhiteSpace = false;
+ @$this->dom->loadHTML($html);
+ }
+ $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
+ }
+
+ /**
+ * Get article title element
+ * @return DOMElement
+ */
+ public function getTitle() {
+ return $this->articleTitle;
+ }
+
+ /**
+ * Get article content element
+ * @return DOMElement
+ */
+ public function getContent() {
+ return $this->articleContent;
+ }
+
+ /**
+ * Runs readability.
+ *
+ * Workflow:
+ * 1. Prep the document by removing script tags, css, etc.
+ * 2. Build readability's DOM tree.
+ * 3. Grab the article content from the current dom tree.
+ * 4. Replace the current DOM tree with the new one.
+ * 5. Read peacefully.
+ *
+ * @return boolean true if we found content, false otherwise
+ **/
+ public function init()
+ {
+ if (!isset($this->dom->documentElement)) return false;
+ $this->removeScripts($this->dom);
+ //die($this->getInnerHTML($this->dom->documentElement));
+
+ // Assume successful outcome
+ $this->success = true;
+
+ $bodyElems = $this->dom->getElementsByTagName('body');
+ if ($bodyElems->length > 0) {
+ if ($this->bodyCache == null) {
+ $this->bodyCache = $bodyElems->item(0)->innerHTML;
+ }
+ if ($this->body == null) {
+ $this->body = $bodyElems->item(0);
+ }
+ }
+
+ $this->prepDocument();
+
+ //die($this->dom->documentElement->parentNode->nodeType);
+ //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
+ //die($this->getInnerHTML($this->dom->documentElement));
+
+ /* Build readability's DOM tree */
+ $overlay = $this->dom->createElement('div');
+ $innerDiv = $this->dom->createElement('div');
+ $articleTitle = $this->getArticleTitle();
+ $articleContent = $this->grabArticle();
+
+ if (!$articleContent) {
+ $this->success = false;
+ $articleContent = $this->dom->createElement('div');
+ $articleContent->setAttribute('id', 'readability-content');
+ $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
+ }
+
+ $overlay->setAttribute('id', 'readOverlay');
+ $innerDiv->setAttribute('id', 'readInner');
+
+ /* Glue the structure of our document together. */
+ $innerDiv->appendChild($articleTitle);
+ $innerDiv->appendChild($articleContent);
+ $overlay->appendChild($innerDiv);
+
+ /* Clear the old HTML, insert the new content. */
+ $this->body->innerHTML = '';
+ $this->body->appendChild($overlay);
+ //document.body.insertBefore(overlay, document.body.firstChild);
+ $this->body->removeAttribute('style');
+
+ $this->postProcessContent($articleContent);
+
+ // Set title and content instance variables
+ $this->articleTitle = $articleTitle;
+ $this->articleContent = $articleContent;
+
+ return $this->success;
+ }
+
+ /**
+ * Debug
+ */
+ protected function dbg($msg) {
+ if ($this->debug) echo '* ',$msg, "\n";
+ }
+
+ /**
+ * Run any post-process modifications to article content as necessary.
+ *
+ * @param DOMElement
+ * @return void
+ */
+ public function postProcessContent($articleContent) {
+ if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) {
+ $this->addFootnotes($articleContent);
+ }
+ }
+
+ /**
+ * Get the article title as an H1.
+ *
+ * @return DOMElement
+ */
+ protected function getArticleTitle() {
+ $curTitle = '';
+ $origTitle = '';
+
+ try {
+ $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
+ } catch(Exception $e) {}
+
+ if (preg_match('/ [\|\-] /', $curTitle))
+ {
+ $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
+
+ if (count(explode(' ', $curTitle)) < 3) {
+ $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
+ }
+ }
+ else if (strpos($curTitle, ': ') !== false)
+ {
+ $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
+
+ if (count(explode(' ', $curTitle)) < 3) {
+ $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
+ }
+ }
+ else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)
+ {
+ $hOnes = $this->dom->getElementsByTagName('h1');
+ if($hOnes->length == 1)
+ {
+ $curTitle = $this->getInnerText($hOnes->item(0));
+ }
+ }
+
+ $curTitle = trim($curTitle);
+
+ if (count(explode(' ', $curTitle)) <= 4) {
+ $curTitle = $origTitle;
+ }
+
+ $articleTitle = $this->dom->createElement('h1');
+ $articleTitle->innerHTML = $curTitle;
+
+ return $articleTitle;
+ }
+
+ /**
+ * Prepare the HTML document for readability to scrape it.
+ * This includes things like stripping javascript, CSS, and handling terrible markup.
+ *
+ * @return void
+ **/
+ protected function prepDocument() {
+ /**
+ * In some cases a body element can't be found (if the HTML is totally hosed for example)
+ * so we create a new body node and append it to the document.
+ */
+ if ($this->body == null)
+ {
+ $this->body = $this->dom->createElement('body');
+ $this->dom->documentElement->appendChild($this->body);
+ }
+ $this->body->setAttribute('id', 'readabilityBody');
+
+ /* Remove all style tags in head */
+ $styleTags = $this->dom->getElementsByTagName('style');
+ for ($i = $styleTags->length-1; $i >= 0; $i--)
+ {
+ $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
+ }
+
+ /* Turn all double br's into p's */
+ /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
+ //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
+ // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
+ // Manipulating innerHTML as it's done in JS is not possible in PHP.
+ }
+
+ /**
+ * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
+ * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
+ *
+ * @return void
+ **/
+ public function addFootnotes($articleContent) {
+ $footnotesWrapper = $this->dom->createElement('div');
+ $footnotesWrapper->setAttribute('id', 'readability-footnotes');
+ $footnotesWrapper->innerHTML = '<h3>References</h3>';
+
+ $articleFootnotes = $this->dom->createElement('ol');
+ $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
+ $footnotesWrapper->appendChild($articleFootnotes);
+
+ $articleLinks = $articleContent->getElementsByTagName('a');
+
+ $linkCount = 0;
+ for ($i = 0; $i < $articleLinks->length; $i++)
+ {
+ $articleLink = $articleLinks->item($i);
+ $footnoteLink = $articleLink->cloneNode(true);
+ $refLink = $this->dom->createElement('a');
+ $footnote = $this->dom->createElement('li');
+ $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
+ if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);
+ //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,
+ $linkText = $this->getInnerText($articleLink);
+
+ if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
+ continue;
+ }
+
+ $linkCount++;
+
+ /** Add a superscript reference after the article link */
+ $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
+ $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
+ $refLink->setAttribute('class', 'readability-DoNotFootnote');
+ $refLink->setAttribute('style', 'color: inherit;');
+
+ //TODO: does this work or should we use DOMNode.isSameNode()?
+ if ($articleLink->parentNode->lastChild == $articleLink) {
+ $articleLink->parentNode->appendChild($refLink);
+ } else {
+ $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
+ }
+
+ $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
+ $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
+
+ $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
+
+ $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);
+ $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
+
+ $footnote->appendChild($footnoteLink);
+ if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';
+
+ $articleFootnotes->appendChild($footnote);
+ }
+
+ if ($linkCount > 0) {
+ $articleContent->appendChild($footnotesWrapper);
+ }
+ }
+
+ /**
+ * Reverts P elements with class 'readability-styled'
+ * to text nodes - which is what they were before.
+ *
+ * @param DOMElement
+ * @return void
+ */
+ function revertReadabilityStyledElements($articleContent) {
+ $xpath = new DOMXPath($articleContent->ownerDocument);
+ $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
+ //$elems = $articleContent->getElementsByTagName('p');
+ for ($i = $elems->length-1; $i >= 0; $i--) {
+ $e = $elems->item($i);
+ $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
+ //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
+ // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
+ //}
+ }
+ }
+
+ /**
+ * Prepare the article node for display. Clean out any inline styles,
+ * iframes, forms, strip extraneous <p> tags, etc.
+ *
+ * @param DOMElement
+ * @return void
+ */
+ function prepArticle($articleContent) {
+ $this->cleanStyles($articleContent);
+ $this->killBreaks($articleContent);
+ if ($this->revertForcedParagraphElements) {
+ $this->revertReadabilityStyledElements($articleContent);
+ }
+
+ /* Clean out junk from the article content */
+ $this->cleanConditionally($articleContent, 'form');
+ $this->clean($articleContent, 'object');
+ $this->clean($articleContent, 'h1');
+
+ /**
+ * If there is only one h2, they are probably using it
+ * as a header and not a subheader, so remove it since we already have a header.
+ ***/
+ if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
+ $this->clean($articleContent, 'h2');
+ }
+ $this->clean($articleContent, 'iframe');
+
+ $this->cleanHeaders($articleContent);
+
+ /* Do these last as the previous stuff may have removed junk that will affect these */
+ $this->cleanConditionally($articleContent, 'table');
+ $this->cleanConditionally($articleContent, 'ul');
+ $this->cleanConditionally($articleContent, 'div');
+
+ /* Remove extra paragraphs */
+ $articleParagraphs = $articleContent->getElementsByTagName('p');
+ for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
+ {
+ $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
+ $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
+ $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
+ $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
+
+ if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
+ {
+ $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
+ }
+ }
+
+ try {
+ $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);
+ //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
+ }
+ catch (Exception $e) {
+ $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
+ }
+ }
+
+ /**
+ * Initialize a node with the readability object. Also checks the
+ * className/id for special names to add to its score.
+ *
+ * @param Element
+ * @return void
+ **/
+ protected function initializeNode($node) {
+ $readability = $this->dom->createAttribute('readability');
+ $readability->value = 0; // this is our contentScore
+ $node->setAttributeNode($readability);
+
+ switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
+ case 'DIV':
+ $readability->value += 5;
+ break;
+
+ case 'PRE':
+ case 'TD':
+ case 'BLOCKQUOTE':
+ $readability->value += 3;
+ break;
+
+ case 'ADDRESS':
+ case 'OL':
+ case 'UL':
+ case 'DL':
+ case 'DD':
+ case 'DT':
+ case 'LI':
+ case 'FORM':
+ $readability->value -= 3;
+ break;
+
+ case 'H1':
+ case 'H2':
+ case 'H3':
+ case 'H4':
+ case 'H5':
+ case 'H6':
+ case 'TH':
+ $readability->value -= 5;
+ break;
+ }
+ $readability->value += $this->getClassWeight($node);
+ }
+
+ /***
+ * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
+ * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
+ *
+ * @return DOMElement
+ **/
+ protected function grabArticle($page=null) {
+ $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
+ if (!$page) $page = $this->dom;
+ $allElements = $page->getElementsByTagName('*');
+ /**
+ * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
+ * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
+ *
+ * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
+ * TODO: Shouldn't this be a reverse traversal?
+ **/
+ $node = null;
+ $nodesToScore = array();
+ for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
+ //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
+ //$node = $targetList->item($nodeIndex);
+ $tagName = strtoupper($node->tagName);
+ /* Remove unlikely candidates */
+ if ($stripUnlikelyCandidates) {
+ $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
+ if (
+ preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
+ !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
+ $tagName != 'BODY'
+ )
+ {
+ $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
+ //$nodesToRemove[] = $node;
+ $node->parentNode->removeChild($node);
+ $nodeIndex--;
+ continue;
+ }
+ }
+
+ if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
+ $nodesToScore[] = $node;
+ }
+
+ /* Turn all divs that don't have children block level elements into p's */
+ if ($tagName == 'DIV') {
+ if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
+ //$this->dbg('Altering div to p');
+ $newNode = $this->dom->createElement('p');
+ try {
+ $newNode->innerHTML = $node->innerHTML;
+ //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
+ $node->parentNode->replaceChild($newNode, $node);
+ $nodeIndex--;
+ $nodesToScore[] = $node; // or $newNode?
+ }
+ catch(Exception $e) {
+ $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
+ }
+ }
+ else
+ {
+ /* EXPERIMENTAL */
+ // TODO: change these p elements back to text nodes after processing
+ for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
+ $childNode = $node->childNodes->item($i);
+ if ($childNode->nodeType == 3) { // XML_TEXT_NODE
+ //$this->dbg('replacing text node with a p tag with the same content.');
+ $p = $this->dom->createElement('p');
+ $p->innerHTML = $childNode->nodeValue;
+ $p->setAttribute('style', 'display: inline;');
+ $p->setAttribute('class', 'readability-styled');
+ $childNode->parentNode->replaceChild($p, $childNode);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Loop through all paragraphs, and assign a score to them based on how content-y they look.
+ * Then add their score to their parent node.
+ *
+ * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
+ **/
+ $candidates = array();
+ for ($pt=0; $pt < count($nodesToScore); $pt++) {
+ $parentNode = $nodesToScore[$pt]->parentNode;
+ // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
+ $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
+ $innerText = $this->getInnerText($nodesToScore[$pt]);
+
+ if (!$parentNode || !isset($parentNode->tagName)) {
+ continue;
+ }
+
+ /* If this paragraph is less than 25 characters, don't even count it. */
+ if(strlen($innerText) < 25) {
+ continue;
+ }
+
+ /* Initialize readability data for the parent. */
+ if (!$parentNode->hasAttribute('readability'))
+ {
+ $this->initializeNode($parentNode);
+ $candidates[] = $parentNode;
+ }
+
+ /* Initialize readability data for the grandparent. */
+ if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
+ {
+ $this->initializeNode($grandParentNode);
+ $candidates[] = $grandParentNode;
+ }
+
+ $contentScore = 0;
+
+ /* Add a point for the paragraph itself as a base. */
+ $contentScore++;
+
+ /* Add points for any commas within this paragraph */
+ $contentScore += count(explode(',', $innerText));
+
+ /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
+ $contentScore += min(floor(strlen($innerText) / 100), 3);
+
+ /* Add the score to the parent. The grandparent gets half. */
+ $parentNode->getAttributeNode('readability')->value += $contentScore;
+
+ if ($grandParentNode) {
+ $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
+ }
+ }
+
+ /**
+ * After we've calculated scores, loop through all of the possible candidate nodes we found
+ * and find the one with the highest score.
+ **/
+ $topCandidate = null;
+ for ($c=0, $cl=count($candidates); $c < $cl; $c++)
+ {
+ /**
+ * Scale the final candidates score based on link density. Good content should have a
+ * relatively small link density (5% or less) and be mostly unaffected by this operation.
+ **/
+ $readability = $candidates[$c]->getAttributeNode('readability');
+ $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
+
+ $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
+
+ if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
+ $topCandidate = $candidates[$c];
+ }
+ }
+
+ /**
+ * If we still have no top candidate, just use the body as a last resort.
+ * We also have to copy the body node so it is something we can modify.
+ **/
+ if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
+ {
+ $topCandidate = $this->dom->createElement('div');
+ if ($page instanceof DOMDocument) {
+ if (!isset($page->documentElement)) {
+ // we don't have a body either? what a mess! :)
+ } else {
+ $topCandidate->innerHTML = $page->documentElement->innerHTML;
+ $page->documentElement->innerHTML = '';
+ $page->documentElement->appendChild($topCandidate);
+ }
+ } else {
+ $topCandidate->innerHTML = $page->innerHTML;
+ $page->innerHTML = '';
+ $page->appendChild($topCandidate);
+ }
+ $this->initializeNode($topCandidate);
+ }
+
+ /**
+ * Now that we have the top candidate, look through its siblings for content that might also be related.
+ * Things like preambles, content split by ads that we removed, etc.
+ **/
+ $articleContent = $this->dom->createElement('div');
+ $articleContent->setAttribute('id', 'readability-content');
+ $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
+ $siblingNodes = $topCandidate->parentNode->childNodes;
+ if (!isset($siblingNodes)) {
+ $siblingNodes = new stdClass;
+ $siblingNodes->length = 0;
+ }
+
+ for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
+ {
+ $siblingNode = $siblingNodes->item($s);
+ $append = false;
+
+ $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
+
+ //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
+
+ if ($siblingNode === $topCandidate)
+ // or if ($siblingNode->isSameNode($topCandidate))
+ {
+ $append = true;
+ }
+
+ $contentBonus = 0;
+ /* Give a bonus if sibling nodes and top candidates have the example same classname */
+ if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
+ $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
+ }
+
+ if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
+ {
+ $append = true;
+ }
+
+ if (strtoupper($siblingNode->nodeName) == 'P') {
+ $linkDensity = $this->getLinkDensity($siblingNode);
+ $nodeContent = $this->getInnerText($siblingNode);
+ $nodeLength = strlen($nodeContent);
+
+ if ($nodeLength > 80 && $linkDensity < 0.25)
+ {
+ $append = true;
+ }
+ else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
+ {
+ $append = true;
+ }
+ }
+
+ if ($append)
+ {
+ $this->dbg('Appending node: ' . $siblingNode->nodeName);
+
+ $nodeToAppend = null;
+ $sibNodeName = strtoupper($siblingNode->nodeName);
+ if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
+ /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
+
+ $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
+ $nodeToAppend = $this->dom->createElement('div');
+ try {
+ $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
+ $nodeToAppend->innerHTML = $siblingNode->innerHTML;
+ }
+ catch(Exception $e)
+ {
+ $this->dbg('Could not alter siblingNode to div, reverting back to original.');
+ $nodeToAppend = $siblingNode;
+ $s--;
+ $sl--;
+ }
+ } else {
+ $nodeToAppend = $siblingNode;
+ $s--;
+ $sl--;
+ }
+
+ /* To ensure a node does not interfere with readability styles, remove its classnames */
+ $nodeToAppend->removeAttribute('class');
+
+ /* Append sibling and subtract from our list because it removes the node when you append to another node */
+ $articleContent->appendChild($nodeToAppend);
+ }
+ }
+
+ /**
+ * So we have all of the content that we need. Now we clean it up for presentation.
+ **/
+ $this->prepArticle($articleContent);
+
+ /**
+ * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
+ * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
+ * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
+ * finding the -right- content.
+ **/
+ if (strlen($this->getInnerText($articleContent, false)) < 250)
+ {
+ // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
+ // in the meantime, we check and create an empty element if it's not there.
+ if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');
+ $this->body->innerHTML = $this->bodyCache;
+
+ if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
+ $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
+ return $this->grabArticle($this->body);
+ }
+ else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
+ $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
+ return $this->grabArticle($this->body);
+ }
+ else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
+ $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
+ return $this->grabArticle($this->body);
+ }
+ else {
+ return false;
+ }
+ }
+ return $articleContent;
+ }
+
+ /**
+ * Remove script tags from document
+ *
+ * @param DOMElement
+ * @return void
+ */
+ public function removeScripts($doc) {
+ $scripts = $doc->getElementsByTagName('script');
+ for($i = $scripts->length-1; $i >= 0; $i--)
+ {
+ $scripts->item($i)->parentNode->removeChild($scripts->item($i));
+ }
+ }
+
+ /**
+ * Get the inner text of a node.
+ * This also strips out any excess whitespace to be found.
+ *
+ * @param DOMElement $
+ * @param boolean $normalizeSpaces (default: true)
+ * @return string
+ **/
+ public function getInnerText($e, $normalizeSpaces=true) {
+ $textContent = '';
+
+ if (!isset($e->textContent) || $e->textContent == '') {
+ return '';
+ }
+
+ $textContent = trim($e->textContent);
+
+ if ($normalizeSpaces) {
+ return preg_replace($this->regexps['normalize'], ' ', $textContent);
+ } else {
+ return $textContent;
+ }
+ }
+
+ /**
+ * Get the number of times a string $s appears in the node $e.
+ *
+ * @param DOMElement $e
+ * @param string - what to count. Default is ","
+ * @return number (integer)
+ **/
+ public function getCharCount($e, $s=',') {
+ return substr_count($this->getInnerText($e), $s);
+ }
+
+ /**
+ * Remove the style attribute on every $e and under.
+ *
+ * @param DOMElement $e
+ * @return void
+ */
+ public function cleanStyles($e) {
+ if (!is_object($e)) return;
+ $elems = $e->getElementsByTagName('*');
+ foreach ($elems as $elem) {
+ $elem->removeAttribute('style');
+ }
+ }
+
+ /**
+ * Get the density of links as a percentage of the content
+ * This is the amount of text that is inside a link divided by the total text in the node.
+ *
+ * @param DOMElement $e
+ * @return number (float)
+ */
+ public function getLinkDensity($e) {
+ $links = $e->getElementsByTagName('a');
+ $textLength = strlen($this->getInnerText($e));
+ $linkLength = 0;
+ for ($i=0, $il=$links->length; $i < $il; $i++)
+ {
+ $linkLength += strlen($this->getInnerText($links->item($i)));
+ }
+ if ($textLength > 0) {
+ return $linkLength / $textLength;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Get an elements class/id weight. Uses regular expressions to tell if this
+ * element looks good or bad.
+ *
+ * @param DOMElement $e
+ * @return number (Integer)
+ */
+ public function getClassWeight($e) {
+ if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
+ return 0;
+ }
+
+ $weight = 0;
+
+ /* Look for a special classname */
+ if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
+ {
+ if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
+ $weight -= 25;
+ }
+ if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
+ $weight += 25;
+ }
+ }
+
+ /* Look for a special ID */
+ if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
+ {
+ if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
+ $weight -= 25;
+ }
+ if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
+ $weight += 25;
+ }
+ }
+ return $weight;
+ }
+
+ /**
+ * Remove extraneous break tags from a node.
+ *
+ * @param DOMElement $node
+ * @return void
+ */
+ public function killBreaks($node) {
+ $html = $node->innerHTML;
+ $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
+ $node->innerHTML = $html;
+ }
+
+ /**
+ * Clean a node of all elements of type "tag".
+ * (Unless it's a youtube/vimeo video. People love movies.)
+ *
+ * Updated 2012-09-18 to preserve youtube/vimeo iframes
+ *
+ * @param DOMElement $e
+ * @param string $tag
+ * @return void
+ */
+ public function clean($e, $tag) {
+ $targetList = $e->getElementsByTagName($tag);
+ $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
+
+ for ($y=$targetList->length-1; $y >= 0; $y--) {
+ /* Allow youtube and vimeo videos through as people usually want to see those. */
+ if ($isEmbed) {
+ $attributeValues = '';
+ for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
+ $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
+ }
+
+ /* First, check the elements attributes to see if any of them contain youtube or vimeo */
+ if (preg_match($this->regexps['video'], $attributeValues)) {
+ continue;
+ }
+
+ /* Then check the elements inside this element for the same. */
+ if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
+ continue;
+ }
+ }
+ $targetList->item($y)->parentNode->removeChild($targetList->item($y));
+ }
+ }
+
+ /**
+ * Clean an element of all tags of type "tag" if they look fishy.
+ * "Fishy" is an algorithm based on content length, classnames,
+ * link density, number of images & embeds, etc.
+ *
+ * @param DOMElement $e
+ * @param string $tag
+ * @return void
+ */
+ public function cleanConditionally($e, $tag) {
+ if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
+ return;
+ }
+
+ $tagsList = $e->getElementsByTagName($tag);
+ $curTagsLength = $tagsList->length;
+
+ /**
+ * Gather counts for other typical elements embedded within.
+ * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
+ *
+ * TODO: Consider taking into account original contentScore here.
+ */
+ for ($i=$curTagsLength-1; $i >= 0; $i--) {
+ $weight = $this->getClassWeight($tagsList->item($i));
+ $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
+
+ $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
+
+ if ($weight + $contentScore < 0) {
+ $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
+ }
+ else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
+ /**
+ * If there are not very many commas, and the number of
+ * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
+ **/
+ $p = $tagsList->item($i)->getElementsByTagName('p')->length;
+ $img = $tagsList->item($i)->getElementsByTagName('img')->length;
+ $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
+ $input = $tagsList->item($i)->getElementsByTagName('input')->length;
+ $a = $tagsList->item($i)->getElementsByTagName('a')->length;
+
+ $embedCount = 0;
+ $embeds = $tagsList->item($i)->getElementsByTagName('embed');
+ for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
+ if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
+ $embedCount++;
+ }
+ }
+ $embeds = $tagsList->item($i)->getElementsByTagName('iframe');
+ for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
+ if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
+ $embedCount++;
+ }
+ }
+
+ $linkDensity = $this->getLinkDensity($tagsList->item($i));
+ $contentLength = strlen($this->getInnerText($tagsList->item($i)));
+ $toRemove = false;
+
+ if ($this->lightClean) {
+ $this->dbg('Light clean...');
+ if ( ($img > $p) && ($img > 4) ) {
+ $this->dbg(' more than 4 images and more image elements than paragraph elements');
+ $toRemove = true;
+ } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+ $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
+ $toRemove = true;
+ } else if ( $input > floor($p/3) ) {
+ $this->dbg(' too many <input> elements');
+ $toRemove = true;
+ } else if ($contentLength < 10 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
+ $this->dbg(' content length less than 10 chars, 0 embeds and either 0 images or more than 2 images');
+ $toRemove = true;
+ } else if($weight < 25 && $linkDensity > 0.2) {
+ $this->dbg(' weight smaller than 25 and link density above 0.2');
+ $toRemove = true;
+ } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
+ $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
+ $toRemove = true;
+ } else if($embedCount > 3) {
+ $this->dbg(' more than 3 embeds');
+ $toRemove = true;
+ }
+ } else {
+ $this->dbg('Standard clean...');
+ if ( $img > $p ) {
+ $this->dbg(' more image elements than paragraph elements');
+ $toRemove = true;
+ } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+ $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
+ $toRemove = true;
+ } else if ( $input > floor($p/3) ) {
+ $this->dbg(' too many <input> elements');
+ $toRemove = true;
+ } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
+ $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
+ $toRemove = true;
+ } else if($weight < 25 && $linkDensity > 0.2) {
+ $this->dbg(' weight smaller than 25 and link density above 0.2');
+ $toRemove = true;
+ } else if($weight >= 25 && $linkDensity > 0.5) {
+ $this->dbg(' weight above 25 but link density greater than 0.5');
+ $toRemove = true;
+ } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
+ $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
+ $toRemove = true;
+ }
+ }
+
+ if ($toRemove) {
+ //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
+ $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
+ }
+ }
+ }
+ }
+
+ /**
+ * Clean out spurious headers from an Element. Checks things like classnames and link density.
+ *
+ * @param DOMElement $e
+ * @return void
+ */
+ public function cleanHeaders($e) {
+ for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
+ $headers = $e->getElementsByTagName('h' . $headerIndex);
+ for ($i=$headers->length-1; $i >=0; $i--) {
+ if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
+ $headers->item($i)->parentNode->removeChild($headers->item($i));
+ }
+ }
+ }
+ }
+
+ public function flagIsActive($flag) {
+ return ($this->flags & $flag) > 0;
+ }
+
+ public function addFlag($flag) {
+ $this->flags = $this->flags | $flag;
+ }
+
+ public function removeFlag($flag) {
+ $this->flags = $this->flags & ~$flag;
+ }
+}
?>
\ No newline at end of file
projects / github / wallabag / wallabag.git / blobdiff