[github/wallabag/wallabag.git] / inc / 3rdparty / libraries / readability / Readability.php

<?php\r
/** \r
* Arc90's Readability ported to PHP for FiveFilters.org\r
* Based on readability.js version 1.7.1 (without multi-page support)\r
* Updated to allow HTML5 parsing with html5lib\r
* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds\r
* ------------------------------------------------------\r
* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js\r
* Arc90's project URL: http://lab.arc90.com/experiments/readability/\r
* JS Source: http://code.google.com/p/arc90labs-readability\r
* Ported by: Keyvan Minoukadeh, http://www.keyvan.net\r
* More information: http://fivefilters.org/content-only/\r
* License: Apache License, Version 2.0\r
* Requires: PHP5\r
* Date: 2012-09-19\r
* \r
* Differences between the PHP port and the original\r
* ------------------------------------------------------\r
* Arc90's Readability is designed to run in the browser. It works on the DOM \r
* tree (the parsed HTML) after the page's CSS styles have been applied and \r
* Javascript code executed. This PHP port does not run inside a browser. \r
* We use PHP's ability to parse HTML to build our DOM tree, but we cannot \r
* rely on CSS or Javascript support. As such, the results will not always \r
* match Arc90's Readability. (For example, if a web page contains CSS style \r
* rules or Javascript code which hide certain HTML elements from display, \r
* Arc90's Readability will dismiss those from consideration but our PHP port, \r
* unable to understand CSS or Javascript, will not know any better.)\r
* \r
* Another significant difference is that the aim of Arc90's Readability is \r
* to re-present the main content block of a given web page so users can \r
* read it more easily in their browsers. Correct identification, clean up, \r
* and separation of the content block is only a part of this process. \r
* This PHP port is only concerned with this part, it does not include code \r
* that relates to presentation in the browser - Arc90 already do \r
* that extremely well, and for PDF output there's FiveFilters.org's \r
* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.\r
* \r
* Finally, this class contains methods that might be useful for developers \r
* working on HTML document fragments. So without deviating too much from \r
* the original code (which I don't want to do because it makes debugging \r
* and updating more difficult), I've tried to make it a little more \r
* developer friendly. You should be able to use the methods here on \r
* existing DOMElement objects without passing an entire HTML document to \r
* be parsed.\r
*/\r
\r
// This class allows us to do JavaScript like assignements to innerHTML\r
require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');\r
\r
// Alternative usage (for testing only!)\r
// uncomment the lines below and call Readability.php in your browser \r
// passing it the URL of the page you'd like content from, e.g.:\r
// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php\r
\r
/*\r
if (!isset($_GET['url']) || $_GET['url'] == '') {\r
	die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');\r
}\r
$url = $_GET['url'];\r
if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;\r
$html = file_get_contents($url);\r
$r = new Readability($html, $url);\r
$r->init();\r
echo $r->articleContent->innerHTML;\r
*/\r
\r
class Readability\r
{\r
	public $version = '1.7.1-without-multi-page';\r
	public $convertLinksToFootnotes = false;\r
	public $revertForcedParagraphElements = true;\r
	public $articleTitle;\r
	public $articleContent;\r
	public $dom;\r
	public $url = null; // optional - URL where HTML was retrieved\r
	public $debug = false;\r
	public $lightClean = true; // preserves more content (experimental) added 2012-09-19\r
	protected $body = null; // \r
	protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later\r
	protected $flags = 7; // 1 | 2 | 4;   // Start with all flags set.\r
	protected $success = false; // indicates whether we were able to extract or not\r
	\r
	/**\r
	* All of the regular expressions in use within readability.\r
	* Defined up here so we don't instantiate them repeatedly in loops.\r
	**/\r
	public $regexps = array(\r
		'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',\r
		'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',\r
		'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',\r
		'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',\r
		'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',\r
		'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',\r
		'replaceFonts' => '/<(\/?)font[^>]*>/i',\r
		// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()\r
		'normalize' => '/\s{2,}/',\r
		'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',\r
		'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',\r
		'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'\r
	);	\r
	\r
	/* constants */\r
	const FLAG_STRIP_UNLIKELYS = 1;\r
	const FLAG_WEIGHT_CLASSES = 2;\r
	const FLAG_CLEAN_CONDITIONALLY = 4;\r
	\r
	/**\r
	* Create instance of Readability\r
	* @param string UTF-8 encoded string\r
	* @param string (optional) URL associated with HTML (used for footnotes)\r
	* @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')\r
	*/	\r
	function __construct($html, $url=null, $parser='libxml')\r
	{\r
		$this->url = $url;\r
		/* Turn all double br's into p's */\r
		$html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);\r
		$html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);\r
		$html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");\r
		if (trim($html) == '') $html = '<html></html>';\r
		if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {\r
			// all good\r
		} else {\r
			$this->dom = new DOMDocument();\r
			$this->dom->preserveWhiteSpace = false;\r
			@$this->dom->loadHTML($html);\r
		}\r
		$this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');\r
	}\r
\r
	/**\r
	* Get article title element\r
	* @return DOMElement\r
	*/\r
	public function getTitle() {\r
		return $this->articleTitle;\r
	}\r
	\r
	/**\r
	* Get article content element\r
	* @return DOMElement\r
	*/\r
	public function getContent() {\r
		return $this->articleContent;\r
	}	\r
	\r
	/**\r
	* Runs readability.\r
	* \r
	* Workflow:\r
	*  1. Prep the document by removing script tags, css, etc.\r
	*  2. Build readability's DOM tree.\r
	*  3. Grab the article content from the current dom tree.\r
	*  4. Replace the current DOM tree with the new one.\r
	*  5. Read peacefully.\r
	*\r
	* @return boolean true if we found content, false otherwise\r
	**/\r
	public function init()\r
	{\r
		if (!isset($this->dom->documentElement)) return false;\r
		$this->removeScripts($this->dom);\r
		//die($this->getInnerHTML($this->dom->documentElement));\r
		\r
		// Assume successful outcome\r
		$this->success = true;\r
\r
		$bodyElems = $this->dom->getElementsByTagName('body');\r
		if ($bodyElems->length > 0) {\r
			if ($this->bodyCache == null) {\r
				$this->bodyCache = $bodyElems->item(0)->innerHTML;\r
			}\r
			if ($this->body == null) {\r
				$this->body = $bodyElems->item(0);\r
			}\r
		}\r
\r
		$this->prepDocument();\r
		\r
		//die($this->dom->documentElement->parentNode->nodeType);\r
		//$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));\r
		//die($this->getInnerHTML($this->dom->documentElement));\r
\r
		/* Build readability's DOM tree */\r
		$overlay        = $this->dom->createElement('div');\r
		$innerDiv       = $this->dom->createElement('div');\r
		$articleTitle   = $this->getArticleTitle();\r
		$articleContent = $this->grabArticle();\r
\r
		if (!$articleContent) {\r
			$this->success = false;\r
			$articleContent = $this->dom->createElement('div');\r
			$articleContent->setAttribute('id', 'readability-content');\r
			$articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';		\r
		}\r
		\r
		$overlay->setAttribute('id', 'readOverlay');\r
		$innerDiv->setAttribute('id', 'readInner');\r
\r
		/* Glue the structure of our document together. */\r
		$innerDiv->appendChild($articleTitle);\r
		$innerDiv->appendChild($articleContent);\r
		$overlay->appendChild($innerDiv);\r
		\r
		/* Clear the old HTML, insert the new content. */\r
		$this->body->innerHTML = '';\r
		$this->body->appendChild($overlay);\r
		//document.body.insertBefore(overlay, document.body.firstChild);\r
		$this->body->removeAttribute('style');\r
\r
		$this->postProcessContent($articleContent);\r
		\r
		// Set title and content instance variables\r
		$this->articleTitle = $articleTitle;\r
		$this->articleContent = $articleContent;\r
		\r
		return $this->success;\r
	}\r
	\r
	/**\r
	* Debug\r
	*/\r
	protected function dbg($msg) {\r
		if ($this->debug) echo '* ',$msg, "\n";\r
	}\r
	\r
	/**\r
	* Run any post-process modifications to article content as necessary.\r
	*\r
	* @param DOMElement\r
	* @return void\r
	*/\r
	public function postProcessContent($articleContent) {\r
		if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { \r
			$this->addFootnotes($articleContent);\r
		}\r
	}\r
	\r
	/**\r
	* Get the article title as an H1.\r
	*\r
	* @return DOMElement\r
	*/\r
	protected function getArticleTitle() {\r
		$curTitle = '';\r
		$origTitle = '';\r
\r
		try {\r
			$curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));\r
		} catch(Exception $e) {}\r
		\r
		if (preg_match('/ [\|\-] /', $curTitle))\r
		{\r
			$curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);\r
			\r
			if (count(explode(' ', $curTitle)) < 3) {\r
				$curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);\r
			}\r
		}\r
		else if (strpos($curTitle, ': ') !== false)\r
		{\r
			$curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);\r
\r
			if (count(explode(' ', $curTitle)) < 3) {\r
				$curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);\r
			}\r
		}\r
		else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)\r
		{\r
			$hOnes = $this->dom->getElementsByTagName('h1');\r
			if($hOnes->length == 1)\r
			{\r
				$curTitle = $this->getInnerText($hOnes->item(0));\r
			}\r
		}\r
\r
		$curTitle = trim($curTitle);\r
\r
		if (count(explode(' ', $curTitle)) <= 4) {\r
			$curTitle = $origTitle;\r
		}\r
		\r
		$articleTitle = $this->dom->createElement('h1');\r
		$articleTitle->innerHTML = $curTitle;\r
		\r
		return $articleTitle;\r
	}\r
	\r
	/**\r
	* Prepare the HTML document for readability to scrape it.\r
	* This includes things like stripping javascript, CSS, and handling terrible markup.\r
	* \r
	* @return void\r
	**/\r
	protected function prepDocument() {\r
		/**\r
		* In some cases a body element can't be found (if the HTML is totally hosed for example)\r
		* so we create a new body node and append it to the document.\r
		*/\r
		if ($this->body == null)\r
		{\r
			$this->body = $this->dom->createElement('body');\r
			$this->dom->documentElement->appendChild($this->body);\r
		}\r
		$this->body->setAttribute('id', 'readabilityBody');\r
\r
		/* Remove all style tags in head */\r
		$styleTags = $this->dom->getElementsByTagName('style');\r
		for ($i = $styleTags->length-1; $i >= 0; $i--)\r
		{\r
			$styleTags->item($i)->parentNode->removeChild($styleTags->item($i));\r
		}\r
\r
		/* Turn all double br's into p's */\r
		/* Note, this is pretty costly as far as processing goes. Maybe optimize later. */\r
		//document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');\r
		// We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.\r
		// Manipulating innerHTML as it's done in JS is not possible in PHP.\r
	}\r
\r
	/**\r
	* For easier reading, convert this document to have footnotes at the bottom rather than inline links.\r
	* @see http://www.roughtype.com/archives/2010/05/experiments_in.php\r
	*\r
	* @return void\r
	**/\r
	public function addFootnotes($articleContent) {\r
		$footnotesWrapper = $this->dom->createElement('div');\r
		$footnotesWrapper->setAttribute('id', 'readability-footnotes');\r
		$footnotesWrapper->innerHTML = '<h3>References</h3>';\r
		\r
		$articleFootnotes = $this->dom->createElement('ol');\r
		$articleFootnotes->setAttribute('id', 'readability-footnotes-list');\r
		$footnotesWrapper->appendChild($articleFootnotes);\r
		\r
		$articleLinks = $articleContent->getElementsByTagName('a');\r
		\r
		$linkCount = 0;\r
		for ($i = 0; $i < $articleLinks->length; $i++)\r
		{\r
			$articleLink  = $articleLinks->item($i);\r
			$footnoteLink = $articleLink->cloneNode(true);\r
			$refLink      = $this->dom->createElement('a');\r
			$footnote     = $this->dom->createElement('li');\r
			$linkDomain   = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);\r
			if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);\r
			//linkDomain   = footnoteLink.host ? footnoteLink.host : document.location.host,\r
			$linkText     = $this->getInnerText($articleLink);\r
			\r
			if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {\r
				continue;\r
			}\r
			\r
			$linkCount++;\r
\r
			/** Add a superscript reference after the article link */\r
			$refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);\r
			$refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';\r
			$refLink->setAttribute('class', 'readability-DoNotFootnote');\r
			$refLink->setAttribute('style', 'color: inherit;');\r
			\r
			//TODO: does this work or should we use DOMNode.isSameNode()?\r
			if ($articleLink->parentNode->lastChild == $articleLink) {\r
				$articleLink->parentNode->appendChild($refLink);\r
			} else {\r
				$articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);\r
			}\r
\r
			$articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');\r
			$articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);\r
\r
			$footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';\r
\r
			$footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);\r
			$footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);\r
			\r
			$footnote->appendChild($footnoteLink);\r
			if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';\r
			\r
			$articleFootnotes->appendChild($footnote);\r
		}\r
\r
		if ($linkCount > 0) {\r
			$articleContent->appendChild($footnotesWrapper);           \r
		}\r
	}\r
\r
	/**\r
	* Reverts P elements with class 'readability-styled'\r
	* to text nodes - which is what they were before.\r
	*\r
	* @param DOMElement\r
	* @return void\r
	*/\r
	function revertReadabilityStyledElements($articleContent) {\r
		$xpath = new DOMXPath($articleContent->ownerDocument);\r
		$elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);\r
		//$elems = $articleContent->getElementsByTagName('p');\r
		for ($i = $elems->length-1; $i >= 0; $i--) {\r
			$e = $elems->item($i);\r
			$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);\r
			//if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {\r
			//	$e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);\r
			//}\r
		}\r
	}\r
	\r
	/**\r
	* Prepare the article node for display. Clean out any inline styles,\r
	* iframes, forms, strip extraneous <p> tags, etc.\r
	*\r
	* @param DOMElement\r
	* @return void\r
	*/\r
	function prepArticle($articleContent) {\r
		$this->cleanStyles($articleContent);\r
		$this->killBreaks($articleContent);\r
		if ($this->revertForcedParagraphElements) {\r
			$this->revertReadabilityStyledElements($articleContent);\r
		}\r
\r
		/* Clean out junk from the article content */\r
		$this->cleanConditionally($articleContent, 'form');\r
		$this->clean($articleContent, 'object');\r
		$this->clean($articleContent, 'h1');\r
\r
		/**\r
		* If there is only one h2, they are probably using it\r
		* as a header and not a subheader, so remove it since we already have a header.\r
		***/\r
		if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {\r
			$this->clean($articleContent, 'h2'); \r
		}\r
		$this->clean($articleContent, 'iframe');\r
\r
		$this->cleanHeaders($articleContent);\r
\r
		/* Do these last as the previous stuff may have removed junk that will affect these */\r
		$this->cleanConditionally($articleContent, 'table');\r
		$this->cleanConditionally($articleContent, 'ul');\r
		$this->cleanConditionally($articleContent, 'div');\r
\r
		/* Remove extra paragraphs */\r
		$articleParagraphs = $articleContent->getElementsByTagName('p');\r
		for ($i = $articleParagraphs->length-1; $i >= 0; $i--)\r
		{\r
			$imgCount    = $articleParagraphs->item($i)->getElementsByTagName('img')->length;\r
			$embedCount  = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;\r
			$objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;\r
			$iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;\r
			\r
			if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')\r
			{\r
				$articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));\r
			}\r
		}\r
\r
		try {\r
			$articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);\r
			//articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');      \r
		}\r
		catch (Exception $e) {\r
			$this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);\r
		}\r
	}\r
	\r
	/**\r
	* Initialize a node with the readability object. Also checks the\r
	* className/id for special names to add to its score.\r
	*\r
	* @param Element\r
	* @return void\r
	**/\r
	protected function initializeNode($node) {\r
		$readability = $this->dom->createAttribute('readability');\r
		$readability->value = 0; // this is our contentScore\r
		$node->setAttributeNode($readability);		         \r
\r
		switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case\r
			case 'DIV':\r
				$readability->value += 5;\r
				break;\r
\r
			case 'PRE':\r
			case 'TD':\r
			case 'BLOCKQUOTE':\r
				$readability->value += 3;\r
				break;\r
				\r
			case 'ADDRESS':\r
			case 'OL':\r
			case 'UL':\r
			case 'DL':\r
			case 'DD':\r
			case 'DT':\r
			case 'LI':\r
			case 'FORM':\r
				$readability->value -= 3;\r
				break;\r
\r
			case 'H1':\r
			case 'H2':\r
			case 'H3':\r
			case 'H4':\r
			case 'H5':\r
			case 'H6':\r
			case 'TH':\r
				$readability->value -= 5;\r
				break;\r
		}\r
		$readability->value += $this->getClassWeight($node);\r
	}\r
	\r
	/***\r
	* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is\r
	*               most likely to be the stuff a user wants to read. Then return it wrapped up in a div.\r
	*\r
	* @return DOMElement\r
	**/\r
	protected function grabArticle($page=null) {\r
		$stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);\r
		if (!$page) $page = $this->dom;\r
		$allElements = $page->getElementsByTagName('*');\r
		/**\r
		* First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs\r
		* into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)\r
		*\r
		* Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5\r
		* TODO: Shouldn't this be a reverse traversal?\r
		**/\r
		$node = null;\r
		$nodesToScore = array();\r
		for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {\r
		//for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {\r
			//$node = $targetList->item($nodeIndex);\r
			$tagName = strtoupper($node->tagName);\r
			/* Remove unlikely candidates */\r
			if ($stripUnlikelyCandidates) {\r
				$unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');\r
				if (\r
					preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&\r
					!preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&\r
					$tagName != 'BODY'\r
				)\r
				{\r
					$this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);\r
					//$nodesToRemove[] = $node;\r
					$node->parentNode->removeChild($node);\r
					$nodeIndex--;\r
					continue;\r
				}               \r
			}\r
\r
			if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {\r
				$nodesToScore[] = $node;\r
			}\r
\r
			/* Turn all divs that don't have children block level elements into p's */\r
			if ($tagName == 'DIV') {\r
				if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {\r
					//$this->dbg('Altering div to p');\r
					$newNode = $this->dom->createElement('p');\r
					try {\r
						$newNode->innerHTML = $node->innerHTML;\r
						//$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);\r
						$node->parentNode->replaceChild($newNode, $node);\r
						$nodeIndex--;\r
						$nodesToScore[] = $node; // or $newNode?\r
					}\r
					catch(Exception $e) {\r
						$this->dbg('Could not alter div to p, reverting back to div.: ' . $e);\r
					}\r
				}\r
				else\r
				{\r
					/* EXPERIMENTAL */\r
					// TODO: change these p elements back to text nodes after processing\r
					for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {\r
						$childNode = $node->childNodes->item($i);\r
						if ($childNode->nodeType == 3) { // XML_TEXT_NODE\r
							//$this->dbg('replacing text node with a p tag with the same content.');\r
							$p = $this->dom->createElement('p');\r
							$p->innerHTML = $childNode->nodeValue;\r
							$p->setAttribute('style', 'display: inline;');\r
							$p->setAttribute('class', 'readability-styled');\r
							$childNode->parentNode->replaceChild($p, $childNode);\r
						}\r
					}\r
				}\r
			}\r
		}\r
		\r
		/**\r
		* Loop through all paragraphs, and assign a score to them based on how content-y they look.\r
		* Then add their score to their parent node.\r
		*\r
		* A score is determined by things like number of commas, class names, etc. Maybe eventually link density.\r
		**/\r
		$candidates = array();\r
		for ($pt=0; $pt < count($nodesToScore); $pt++) {\r
			$parentNode      = $nodesToScore[$pt]->parentNode;\r
			// $grandParentNode = $parentNode ? $parentNode->parentNode : null;\r
			$grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);\r
			$innerText       = $this->getInnerText($nodesToScore[$pt]);\r
\r
			if (!$parentNode || !isset($parentNode->tagName)) {\r
				continue;\r
			}\r
\r
			/* If this paragraph is less than 25 characters, don't even count it. */\r
			if(strlen($innerText) < 25) {\r
				continue;\r
			}\r
\r
			/* Initialize readability data for the parent. */\r
			if (!$parentNode->hasAttribute('readability')) \r
			{\r
				$this->initializeNode($parentNode);\r
				$candidates[] = $parentNode;\r
			}\r
\r
			/* Initialize readability data for the grandparent. */\r
			if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))\r
			{\r
				$this->initializeNode($grandParentNode);\r
				$candidates[] = $grandParentNode;\r
			}\r
\r
			$contentScore = 0;\r
\r
			/* Add a point for the paragraph itself as a base. */\r
			$contentScore++;\r
\r
			/* Add points for any commas within this paragraph */\r
			$contentScore += count(explode(',', $innerText));\r
			\r
			/* For every 100 characters in this paragraph, add another point. Up to 3 points. */\r
			$contentScore += min(floor(strlen($innerText) / 100), 3);\r
			\r
			/* Add the score to the parent. The grandparent gets half. */\r
			$parentNode->getAttributeNode('readability')->value += $contentScore;\r
\r
			if ($grandParentNode) {\r
				$grandParentNode->getAttributeNode('readability')->value += $contentScore/2;             \r
			}\r
		}\r
\r
		/**\r
		* After we've calculated scores, loop through all of the possible candidate nodes we found\r
		* and find the one with the highest score.\r
		**/\r
		$topCandidate = null;\r
		for ($c=0, $cl=count($candidates); $c < $cl; $c++)\r
		{\r
			/**\r
			* Scale the final candidates score based on link density. Good content should have a\r
			* relatively small link density (5% or less) and be mostly unaffected by this operation.\r
			**/\r
			$readability = $candidates[$c]->getAttributeNode('readability');\r
			$readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));\r
\r
			$this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);\r
\r
			if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {\r
				$topCandidate = $candidates[$c];\r
			}\r
		}\r
\r
		/**\r
		* If we still have no top candidate, just use the body as a last resort.\r
		* We also have to copy the body node so it is something we can modify.\r
		**/\r
		if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')\r
		{\r
			$topCandidate = $this->dom->createElement('div');\r
			if ($page instanceof DOMDocument) {\r
				if (!isset($page->documentElement)) {\r
					// we don't have a body either? what a mess! :)\r
				} else {\r
					$topCandidate->innerHTML = $page->documentElement->innerHTML;\r
					$page->documentElement->innerHTML = '';\r
					$page->documentElement->appendChild($topCandidate);\r
				}\r
			} else {\r
				$topCandidate->innerHTML = $page->innerHTML;\r
				$page->innerHTML = '';\r
				$page->appendChild($topCandidate);\r
			}\r
			$this->initializeNode($topCandidate);\r
		}\r
\r
		/**\r
		* Now that we have the top candidate, look through its siblings for content that might also be related.\r
		* Things like preambles, content split by ads that we removed, etc.\r
		**/\r
		$articleContent        = $this->dom->createElement('div');\r
		$articleContent->setAttribute('id', 'readability-content');\r
		$siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);\r
		$siblingNodes          = $topCandidate->parentNode->childNodes;\r
		if (!isset($siblingNodes)) {\r
			$siblingNodes = new stdClass;\r
			$siblingNodes->length = 0;\r
		}\r
\r
		for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)\r
		{\r
			$siblingNode = $siblingNodes->item($s);\r
			$append      = false;\r
\r
			$this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));\r
\r
			//dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));\r
\r
			if ($siblingNode === $topCandidate)\r
			// or if ($siblingNode->isSameNode($topCandidate))\r
			{\r
				$append = true;\r
			}\r
\r
			$contentBonus = 0;\r
			/* Give a bonus if sibling nodes and top candidates have the example same classname */\r
			if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {\r
				$contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;\r
			}\r
\r
			if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)\r
			{\r
				$append = true;\r
			}\r
			\r
			if (strtoupper($siblingNode->nodeName) == 'P') {\r
				$linkDensity = $this->getLinkDensity($siblingNode);\r
				$nodeContent = $this->getInnerText($siblingNode);\r
				$nodeLength  = strlen($nodeContent);\r
				\r
				if ($nodeLength > 80 && $linkDensity < 0.25)\r
				{\r
					$append = true;\r
				}\r
				else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))\r
				{\r
					$append = true;\r
				}\r
			}\r
\r
			if ($append)\r
			{\r
				$this->dbg('Appending node: ' . $siblingNode->nodeName);\r
\r
				$nodeToAppend = null;\r
				$sibNodeName = strtoupper($siblingNode->nodeName);\r
				if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {\r
					/* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */\r
					\r
					$this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');\r
					$nodeToAppend = $this->dom->createElement('div');\r
					try {\r
						$nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));\r
						$nodeToAppend->innerHTML = $siblingNode->innerHTML;\r
					}\r
					catch(Exception $e)\r
					{\r
						$this->dbg('Could not alter siblingNode to div, reverting back to original.');\r
						$nodeToAppend = $siblingNode;\r
						$s--;\r
						$sl--;\r
					}\r
				} else {\r
					$nodeToAppend = $siblingNode;\r
					$s--;\r
					$sl--;\r
				}\r
				\r
				/* To ensure a node does not interfere with readability styles, remove its classnames */\r
				$nodeToAppend->removeAttribute('class');\r
\r
				/* Append sibling and subtract from our list because it removes the node when you append to another node */\r
				$articleContent->appendChild($nodeToAppend);\r
			}\r
		}\r
\r
		/**\r
		* So we have all of the content that we need. Now we clean it up for presentation.\r
		**/\r
		$this->prepArticle($articleContent);\r
\r
		/**\r
		* Now that we've gone through the full algorithm, check to see if we got any meaningful content.\r
		* If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher\r
		* likelihood of finding the content, and the sieve approach gives us a higher likelihood of\r
		* finding the -right- content.\r
		**/\r
		if (strlen($this->getInnerText($articleContent, false)) < 250)\r
		{\r
			// TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7\r
			// in the meantime, we check and create an empty element if it's not there.\r
			if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');\r
			$this->body->innerHTML = $this->bodyCache;\r
			\r
			if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {\r
				$this->removeFlag(self::FLAG_STRIP_UNLIKELYS);\r
				return $this->grabArticle($this->body);\r
			}\r
			else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {\r
				$this->removeFlag(self::FLAG_WEIGHT_CLASSES);\r
				return $this->grabArticle($this->body);              \r
			}\r
			else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {\r
				$this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);\r
				return $this->grabArticle($this->body);\r
			}\r
			else {\r
				return false;\r
			}\r
		}\r
		return $articleContent;\r
	}\r
	\r
	/**\r
	* Remove script tags from document\r
	*\r
	* @param DOMElement\r
	* @return void\r
	*/\r
	public function removeScripts($doc) {\r
		$scripts = $doc->getElementsByTagName('script');\r
		for($i = $scripts->length-1; $i >= 0; $i--)\r
		{\r
			$scripts->item($i)->parentNode->removeChild($scripts->item($i));\r
		}\r
	}\r
	\r
	/**\r
	* Get the inner text of a node.\r
	* This also strips out any excess whitespace to be found.\r
	*\r
	* @param DOMElement $\r
	* @param boolean $normalizeSpaces (default: true)\r
	* @return string\r
	**/\r
	public function getInnerText($e, $normalizeSpaces=true) {\r
		$textContent = '';\r
\r
		if (!isset($e->textContent) || $e->textContent == '') {\r
			return '';\r
		}\r
\r
		$textContent = trim($e->textContent);\r
\r
		if ($normalizeSpaces) {\r
			return preg_replace($this->regexps['normalize'], ' ', $textContent);\r
		} else {\r
			return $textContent;\r
		}\r
	}\r
\r
	/**\r
	* Get the number of times a string $s appears in the node $e.\r
	*\r
	* @param DOMElement $e\r
	* @param string - what to count. Default is ","\r
	* @return number (integer)\r
	**/\r
	public function getCharCount($e, $s=',') {\r
		return substr_count($this->getInnerText($e), $s);\r
	}\r
\r
	/**\r
	* Remove the style attribute on every $e and under.\r
	*\r
	* @param DOMElement $e\r
	* @return void\r
	*/\r
	public function cleanStyles($e) {\r
		if (!is_object($e)) return;\r
		$elems = $e->getElementsByTagName('*');\r
		foreach ($elems as $elem) {\r
			$elem->removeAttribute('style');\r
		}\r
	}\r
	\r
	/**\r
	* Get the density of links as a percentage of the content\r
	* This is the amount of text that is inside a link divided by the total text in the node.\r
	* \r
	* @param DOMElement $e\r
	* @return number (float)\r
	*/\r
	public function getLinkDensity($e) {\r
		$links      = $e->getElementsByTagName('a');\r
		$textLength = strlen($this->getInnerText($e));\r
		$linkLength = 0;\r
		for ($i=0, $il=$links->length; $i < $il; $i++)\r
		{\r
			$linkLength += strlen($this->getInnerText($links->item($i)));\r
		}\r
		if ($textLength > 0) {\r
			return $linkLength / $textLength;\r
		} else {\r
			return 0;\r
		}\r
	}\r
	\r
	/**\r
	* Get an elements class/id weight. Uses regular expressions to tell if this \r
	* element looks good or bad.\r
	*\r
	* @param DOMElement $e\r
	* @return number (Integer)\r
	*/\r
	public function getClassWeight($e) {\r
		if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {\r
			return 0;\r
		}\r
\r
		$weight = 0;\r
\r
		/* Look for a special classname */\r
		if ($e->hasAttribute('class') && $e->getAttribute('class') != '')\r
		{\r
			if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {\r
				$weight -= 25;\r
			}\r
			if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {\r
				$weight += 25;\r
			}\r
		}\r
\r
		/* Look for a special ID */\r
		if ($e->hasAttribute('id') && $e->getAttribute('id') != '')\r
		{\r
			if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {\r
				$weight -= 25;\r
			}\r
			if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {\r
				$weight += 25;\r
			}\r
		}\r
		return $weight;\r
	}\r
\r
	/**\r
	* Remove extraneous break tags from a node.\r
	*\r
	* @param DOMElement $node\r
	* @return void\r
	*/\r
	public function killBreaks($node) {\r
		$html = $node->innerHTML;\r
		$html = preg_replace($this->regexps['killBreaks'], '<br />', $html);\r
		$node->innerHTML = $html;\r
	}\r
\r
	/**\r
	* Clean a node of all elements of type "tag".\r
	* (Unless it's a youtube/vimeo video. People love movies.)\r
	*\r
	* Updated 2012-09-18 to preserve youtube/vimeo iframes\r
	*\r
	* @param DOMElement $e\r
	* @param string $tag\r
	* @return void\r
	*/\r
	public function clean($e, $tag) {\r
		$targetList = $e->getElementsByTagName($tag);\r
		$isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');\r
		\r
		for ($y=$targetList->length-1; $y >= 0; $y--) {\r
			/* Allow youtube and vimeo videos through as people usually want to see those. */\r
			if ($isEmbed) {\r
				$attributeValues = '';\r
				for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {\r
					$attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)\r
				}\r
				\r
				/* First, check the elements attributes to see if any of them contain youtube or vimeo */\r
				if (preg_match($this->regexps['video'], $attributeValues)) {\r
					continue;\r
				}\r
\r
				/* Then check the elements inside this element for the same. */\r
				if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {\r
					continue;\r
				}\r
			}\r
			$targetList->item($y)->parentNode->removeChild($targetList->item($y));\r
		}\r
	}\r
	\r
	/**\r
	* Clean an element of all tags of type "tag" if they look fishy.\r
	* "Fishy" is an algorithm based on content length, classnames, \r
	* link density, number of images & embeds, etc.\r
	*\r
	* @param DOMElement $e\r
	* @param string $tag\r
	* @return void\r
	*/\r
	public function cleanConditionally($e, $tag) {\r
		if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {\r
			return;\r
		}\r
\r
		$tagsList = $e->getElementsByTagName($tag);\r
		$curTagsLength = $tagsList->length;\r
\r
		/**\r
		* Gather counts for other typical elements embedded within.\r
		* Traverse backwards so we can remove nodes at the same time without effecting the traversal.\r
		*\r
		* TODO: Consider taking into account original contentScore here.\r
		*/\r
		for ($i=$curTagsLength-1; $i >= 0; $i--) {\r
			$weight = $this->getClassWeight($tagsList->item($i));\r
			$contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;\r
			\r
			$this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));\r
\r
			if ($weight + $contentScore < 0) {\r
				$tagsList->item($i)->parentNode->removeChild($tagsList->item($i));\r
			}\r
			else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {\r
				/**\r
				* If there are not very many commas, and the number of\r
				* non-paragraph elements is more than paragraphs or other ominous signs, remove the element.\r
				**/\r
				$p      = $tagsList->item($i)->getElementsByTagName('p')->length;\r
				$img    = $tagsList->item($i)->getElementsByTagName('img')->length;\r
				$li     = $tagsList->item($i)->getElementsByTagName('li')->length-100;\r
				$input  = $tagsList->item($i)->getElementsByTagName('input')->length;\r
				$a 		= $tagsList->item($i)->getElementsByTagName('a')->length;\r
\r
				$embedCount = 0;\r
				$embeds = $tagsList->item($i)->getElementsByTagName('embed');\r
				for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {\r
					if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {\r
						$embedCount++; \r
					}\r
				}\r
				$embeds = $tagsList->item($i)->getElementsByTagName('iframe');\r
				for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {\r
					if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {\r
						$embedCount++; \r
					}\r
				}\r
\r
				$linkDensity   = $this->getLinkDensity($tagsList->item($i));\r
				$contentLength = strlen($this->getInnerText($tagsList->item($i)));\r
				$toRemove      = false;\r
\r
				if ($this->lightClean) {\r
					$this->dbg('Light clean...');\r
					if ( ($img > $p) && ($img > 4) ) {\r
						$this->dbg(' more than 4 images and more image elements than paragraph elements');\r
						$toRemove = true;\r
					} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {\r
						$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');\r
						$toRemove = true;\r
					} else if ( $input > floor($p/3) ) {\r
						$this->dbg(' too many <input> elements');\r
						$toRemove = true; \r
					} else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {\r
						$this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');\r
						$toRemove = true;\r
					} else if($weight < 25 && $linkDensity > 0.2) {\r
						$this->dbg(' weight smaller than 25 and link density above 0.2');\r
						$toRemove = true;\r
					} else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {\r
						$this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');\r
						$toRemove = true;\r
					} else if($embedCount > 3) {\r
						$this->dbg(' more than 3 embeds');\r
						$toRemove = true;\r
					}\r
				} else {\r
					$this->dbg('Standard clean...');\r
					if ( $img > $p ) {\r
						$this->dbg(' more image elements than paragraph elements');\r
						$toRemove = true;\r
					} else if ($li > $p && $tag != 'ul' && $tag != 'ol') {\r
						$this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');\r
						$toRemove = true;\r
					} else if ( $input > floor($p/3) ) {\r
						$this->dbg(' too many <input> elements');\r
						$toRemove = true; \r
					} else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {\r
						$this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');\r
						$toRemove = true;\r
					} else if($weight < 25 && $linkDensity > 0.2) {\r
						$this->dbg(' weight smaller than 25 and link density above 0.2');\r
						$toRemove = true;\r
					} else if($weight >= 25 && $linkDensity > 0.5) {\r
						$this->dbg(' weight above 25 but link density greater than 0.5');\r
						$toRemove = true;\r
					} else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {\r
						$this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');\r
						$toRemove = true;\r
					}\r
				}\r
\r
				if ($toRemove) {\r
					//$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);\r
					$tagsList->item($i)->parentNode->removeChild($tagsList->item($i));\r
				}\r
			}\r
		}\r
	}\r
\r
	/**\r
	* Clean out spurious headers from an Element. Checks things like classnames and link density.\r
	*\r
	* @param DOMElement $e\r
	* @return void\r
	*/\r
	public function cleanHeaders($e) {\r
		for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {\r
			$headers = $e->getElementsByTagName('h' . $headerIndex);\r
			for ($i=$headers->length-1; $i >=0; $i--) {\r
				if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {\r
					$headers->item($i)->parentNode->removeChild($headers->item($i));\r
				}\r
			}\r
		}\r
	}\r
\r
	public function flagIsActive($flag) {\r
		return ($this->flags & $flag) > 0;\r
	}\r
	\r
	public function addFlag($flag) {\r
		$this->flags = $this->flags | $flag;\r
	}\r
	\r
	public function removeFlag($flag) {\r
		$this->flags = $this->flags & ~$flag;\r
	}\r
}\r
?>