aboutsummaryrefslogtreecommitdiffhomepage
path: root/inc/3rdparty/libraries/readability/Readability.php
diff options
context:
space:
mode:
Diffstat (limited to 'inc/3rdparty/libraries/readability/Readability.php')
-rwxr-xr-xinc/3rdparty/libraries/readability/Readability.php1152
1 files changed, 0 insertions, 1152 deletions
diff --git a/inc/3rdparty/libraries/readability/Readability.php b/inc/3rdparty/libraries/readability/Readability.php
deleted file mode 100755
index a30012ce..00000000
--- a/inc/3rdparty/libraries/readability/Readability.php
+++ /dev/null
@@ -1,1152 +0,0 @@
1<?php
2/**
3* Arc90's Readability ported to PHP for FiveFilters.org
4* Based on readability.js version 1.7.1 (without multi-page support)
5* Updated to allow HTML5 parsing with html5lib
6* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
7* ------------------------------------------------------
8* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
9* Arc90's project URL: http://lab.arc90.com/experiments/readability/
10* JS Source: http://code.google.com/p/arc90labs-readability
11* Ported by: Keyvan Minoukadeh, http://www.keyvan.net
12* More information: http://fivefilters.org/content-only/
13* License: Apache License, Version 2.0
14* Requires: PHP5
15* Date: 2012-09-19
16*
17* Differences between the PHP port and the original
18* ------------------------------------------------------
19* Arc90's Readability is designed to run in the browser. It works on the DOM
20* tree (the parsed HTML) after the page's CSS styles have been applied and
21* Javascript code executed. This PHP port does not run inside a browser.
22* We use PHP's ability to parse HTML to build our DOM tree, but we cannot
23* rely on CSS or Javascript support. As such, the results will not always
24* match Arc90's Readability. (For example, if a web page contains CSS style
25* rules or Javascript code which hide certain HTML elements from display,
26* Arc90's Readability will dismiss those from consideration but our PHP port,
27* unable to understand CSS or Javascript, will not know any better.)
28*
29* Another significant difference is that the aim of Arc90's Readability is
30* to re-present the main content block of a given web page so users can
31* read it more easily in their browsers. Correct identification, clean up,
32* and separation of the content block is only a part of this process.
33* This PHP port is only concerned with this part, it does not include code
34* that relates to presentation in the browser - Arc90 already do
35* that extremely well, and for PDF output there's FiveFilters.org's
36* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
37*
38* Finally, this class contains methods that might be useful for developers
39* working on HTML document fragments. So without deviating too much from
40* the original code (which I don't want to do because it makes debugging
41* and updating more difficult), I've tried to make it a little more
42* developer friendly. You should be able to use the methods here on
43* existing DOMElement objects without passing an entire HTML document to
44* be parsed.
45*/
46
47// This class allows us to do JavaScript like assignements to innerHTML
48require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
49libxml_use_internal_errors(true);
50
51// Alternative usage (for testing only!)
52// uncomment the lines below and call Readability.php in your browser
53// passing it the URL of the page you'd like content from, e.g.:
54// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
55
56/*
57if (!isset($_GET['url']) || $_GET['url'] == '') {
58 die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
59}
60$url = $_GET['url'];
61if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
62$html = file_get_contents($url);
63$r = new Readability($html, $url);
64$r->init();
65echo $r->articleContent->innerHTML;
66*/
67
68class Readability
69{
70 public $version = '1.7.1-without-multi-page';
71 public $convertLinksToFootnotes = false;
72 public $revertForcedParagraphElements = true;
73 public $articleTitle;
74 public $articleContent;
75 public $dom;
76 public $url = null; // optional - URL where HTML was retrieved
77 public $debug = false;
78 public $lightClean = true; // preserves more content (experimental) added 2012-09-19
79 protected $body = null; //
80 protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
81 protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
82 protected $success = false; // indicates whether we were able to extract or not
83
84 /**
85 * All of the regular expressions in use within readability.
86 * Defined up here so we don't instantiate them repeatedly in loops.
87 **/
88 public $regexps = array(
89 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
90 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
91 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
92 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
93 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
94 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
95 'replaceFonts' => '/<(\/?)font[^>]*>/i',
96 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
97 'normalize' => '/\s{2,}/',
98 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
99 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
100 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
101 );
102
103 /* constants */
104 const FLAG_STRIP_UNLIKELYS = 1;
105 const FLAG_WEIGHT_CLASSES = 2;
106 const FLAG_CLEAN_CONDITIONALLY = 4;
107
108 /**
109 * Create instance of Readability
110 * @param string UTF-8 encoded string
111 * @param string (optional) URL associated with HTML (used for footnotes)
112 * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
113 */
114 function __construct($html, $url=null, $parser='libxml')
115 {
116 $this->url = $url;
117 /* Turn all double br's into p's */
118 $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
119 $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
120 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
121 if (trim($html) == '') $html = '<html></html>';
122 if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
123 // all good
124 } else {
125 $this->dom = new DOMDocument();
126 $this->dom->preserveWhiteSpace = false;
127 @$this->dom->loadHTML($html);
128 }
129 $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
130 }
131
132 /**
133 * Get article title element
134 * @return DOMElement
135 */
136 public function getTitle() {
137 return $this->articleTitle;
138 }
139
140 /**
141 * Get article content element
142 * @return DOMElement
143 */
144 public function getContent() {
145 return $this->articleContent;
146 }
147
148 /**
149 * Runs readability.
150 *
151 * Workflow:
152 * 1. Prep the document by removing script tags, css, etc.
153 * 2. Build readability's DOM tree.
154 * 3. Grab the article content from the current dom tree.
155 * 4. Replace the current DOM tree with the new one.
156 * 5. Read peacefully.
157 *
158 * @return boolean true if we found content, false otherwise
159 **/
160 public function init()
161 {
162 if (!isset($this->dom->documentElement)) return false;
163 $this->removeScripts($this->dom);
164 //die($this->getInnerHTML($this->dom->documentElement));
165
166 // Assume successful outcome
167 $this->success = true;
168
169 $bodyElems = $this->dom->getElementsByTagName('body');
170 if ($bodyElems->length > 0) {
171 if ($this->bodyCache == null) {
172 $this->bodyCache = $bodyElems->item(0)->innerHTML;
173 }
174 if ($this->body == null) {
175 $this->body = $bodyElems->item(0);
176 }
177 }
178
179 $this->prepDocument();
180
181 //die($this->dom->documentElement->parentNode->nodeType);
182 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
183 //die($this->getInnerHTML($this->dom->documentElement));
184
185 /* Build readability's DOM tree */
186 $overlay = $this->dom->createElement('div');
187 $innerDiv = $this->dom->createElement('div');
188 $articleTitle = $this->getArticleTitle();
189 $articleContent = $this->grabArticle();
190
191 if (!$articleContent) {
192 $this->success = false;
193 $articleContent = $this->dom->createElement('div');
194 $articleContent->setAttribute('id', 'readability-content');
195 $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
196 }
197
198 $overlay->setAttribute('id', 'readOverlay');
199 $innerDiv->setAttribute('id', 'readInner');
200
201 /* Glue the structure of our document together. */
202 $innerDiv->appendChild($articleTitle);
203 $innerDiv->appendChild($articleContent);
204 $overlay->appendChild($innerDiv);
205
206 /* Clear the old HTML, insert the new content. */
207 $this->body->innerHTML = '';
208 $this->body->appendChild($overlay);
209 //document.body.insertBefore(overlay, document.body.firstChild);
210 $this->body->removeAttribute('style');
211
212 $this->postProcessContent($articleContent);
213
214 // Set title and content instance variables
215 $this->articleTitle = $articleTitle;
216 $this->articleContent = $articleContent;
217
218 return $this->success;
219 }
220
221 /**
222 * Debug
223 */
224 protected function dbg($msg) {
225 if ($this->debug) echo '* ',$msg, "\n";
226 }
227
228 /**
229 * Run any post-process modifications to article content as necessary.
230 *
231 * @param DOMElement
232 * @return void
233 */
234 public function postProcessContent($articleContent) {
235 if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) {
236 $this->addFootnotes($articleContent);
237 }
238 }
239
240 /**
241 * Get the article title as an H1.
242 *
243 * @return DOMElement
244 */
245 protected function getArticleTitle() {
246 $curTitle = '';
247 $origTitle = '';
248
249 try {
250 $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
251 } catch(Exception $e) {}
252
253 if (preg_match('/ [\|\-] /', $curTitle))
254 {
255 $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
256
257 if (count(explode(' ', $curTitle)) < 3) {
258 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
259 }
260 }
261 else if (strpos($curTitle, ': ') !== false)
262 {
263 $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
264
265 if (count(explode(' ', $curTitle)) < 3) {
266 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
267 }
268 }
269 else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)
270 {
271 $hOnes = $this->dom->getElementsByTagName('h1');
272 if($hOnes->length == 1)
273 {
274 $curTitle = $this->getInnerText($hOnes->item(0));
275 }
276 }
277
278 $curTitle = trim($curTitle);
279
280 if (count(explode(' ', $curTitle)) <= 4) {
281 $curTitle = $origTitle;
282 }
283
284 $articleTitle = $this->dom->createElement('h1');
285 $articleTitle->innerHTML = $curTitle;
286
287 return $articleTitle;
288 }
289
290 /**
291 * Prepare the HTML document for readability to scrape it.
292 * This includes things like stripping javascript, CSS, and handling terrible markup.
293 *
294 * @return void
295 **/
296 protected function prepDocument() {
297 /**
298 * In some cases a body element can't be found (if the HTML is totally hosed for example)
299 * so we create a new body node and append it to the document.
300 */
301 if ($this->body == null)
302 {
303 $this->body = $this->dom->createElement('body');
304 $this->dom->documentElement->appendChild($this->body);
305 }
306 $this->body->setAttribute('id', 'readabilityBody');
307
308 /* Remove all style tags in head */
309 $styleTags = $this->dom->getElementsByTagName('style');
310 for ($i = $styleTags->length-1; $i >= 0; $i--)
311 {
312 $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
313 }
314
315 /* Turn all double br's into p's */
316 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
317 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
318 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
319 // Manipulating innerHTML as it's done in JS is not possible in PHP.
320 }
321
322 /**
323 * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
324 * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
325 *
326 * @return void
327 **/
328 public function addFootnotes($articleContent) {
329 $footnotesWrapper = $this->dom->createElement('div');
330 $footnotesWrapper->setAttribute('id', 'readability-footnotes');
331 $footnotesWrapper->innerHTML = '<h3>References</h3>';
332
333 $articleFootnotes = $this->dom->createElement('ol');
334 $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
335 $footnotesWrapper->appendChild($articleFootnotes);
336
337 $articleLinks = $articleContent->getElementsByTagName('a');
338
339 $linkCount = 0;
340 for ($i = 0; $i < $articleLinks->length; $i++)
341 {
342 $articleLink = $articleLinks->item($i);
343 $footnoteLink = $articleLink->cloneNode(true);
344 $refLink = $this->dom->createElement('a');
345 $footnote = $this->dom->createElement('li');
346 $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
347 if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);
348 //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,
349 $linkText = $this->getInnerText($articleLink);
350
351 if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
352 continue;
353 }
354
355 $linkCount++;
356
357 /** Add a superscript reference after the article link */
358 $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
359 $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
360 $refLink->setAttribute('class', 'readability-DoNotFootnote');
361 $refLink->setAttribute('style', 'color: inherit;');
362
363 //TODO: does this work or should we use DOMNode.isSameNode()?
364 if ($articleLink->parentNode->lastChild == $articleLink) {
365 $articleLink->parentNode->appendChild($refLink);
366 } else {
367 $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
368 }
369
370 $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
371 $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
372
373 $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
374
375 $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);
376 $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
377
378 $footnote->appendChild($footnoteLink);
379 if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';
380
381 $articleFootnotes->appendChild($footnote);
382 }
383
384 if ($linkCount > 0) {
385 $articleContent->appendChild($footnotesWrapper);
386 }
387 }
388
389 /**
390 * Reverts P elements with class 'readability-styled'
391 * to text nodes - which is what they were before.
392 *
393 * @param DOMElement
394 * @return void
395 */
396 function revertReadabilityStyledElements($articleContent) {
397 $xpath = new DOMXPath($articleContent->ownerDocument);
398 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
399 //$elems = $articleContent->getElementsByTagName('p');
400 for ($i = $elems->length-1; $i >= 0; $i--) {
401 $e = $elems->item($i);
402 $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
403 //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
404 // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
405 //}
406 }
407 }
408
409 /**
410 * Prepare the article node for display. Clean out any inline styles,
411 * iframes, forms, strip extraneous <p> tags, etc.
412 *
413 * @param DOMElement
414 * @return void
415 */
416 function prepArticle($articleContent) {
417 $this->cleanStyles($articleContent);
418 $this->killBreaks($articleContent);
419 if ($this->revertForcedParagraphElements) {
420 $this->revertReadabilityStyledElements($articleContent);
421 }
422
423 /* Clean out junk from the article content */
424 $this->cleanConditionally($articleContent, 'form');
425 $this->clean($articleContent, 'object');
426 $this->clean($articleContent, 'h1');
427
428 /**
429 * If there is only one h2, they are probably using it
430 * as a header and not a subheader, so remove it since we already have a header.
431 ***/
432 if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
433 $this->clean($articleContent, 'h2');
434 }
435 $this->clean($articleContent, 'iframe');
436
437 $this->cleanHeaders($articleContent);
438
439 /* Do these last as the previous stuff may have removed junk that will affect these */
440 $this->cleanConditionally($articleContent, 'table');
441 $this->cleanConditionally($articleContent, 'ul');
442 $this->cleanConditionally($articleContent, 'div');
443
444 /* Remove extra paragraphs */
445 $articleParagraphs = $articleContent->getElementsByTagName('p');
446 for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
447 {
448 $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
449 $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
450 $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
451 $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
452
453 if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
454 {
455 $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
456 }
457 }
458
459 try {
460 $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);
461 //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
462 }
463 catch (Exception $e) {
464 $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
465 }
466 }
467
468 /**
469 * Initialize a node with the readability object. Also checks the
470 * className/id for special names to add to its score.
471 *
472 * @param Element
473 * @return void
474 **/
475 protected function initializeNode($node) {
476 $readability = $this->dom->createAttribute('readability');
477 $readability->value = 0; // this is our contentScore
478 $node->setAttributeNode($readability);
479
480 switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
481 case 'DIV':
482 $readability->value += 5;
483 break;
484
485 case 'PRE':
486 case 'TD':
487 case 'BLOCKQUOTE':
488 $readability->value += 3;
489 break;
490
491 case 'ADDRESS':
492 case 'OL':
493 case 'UL':
494 case 'DL':
495 case 'DD':
496 case 'DT':
497 case 'LI':
498 case 'FORM':
499 $readability->value -= 3;
500 break;
501
502 case 'H1':
503 case 'H2':
504 case 'H3':
505 case 'H4':
506 case 'H5':
507 case 'H6':
508 case 'TH':
509 $readability->value -= 5;
510 break;
511 }
512 $readability->value += $this->getClassWeight($node);
513 }
514
515 /***
516 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
517 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
518 *
519 * @return DOMElement
520 **/
521 protected function grabArticle($page=null) {
522 $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
523 if (!$page) $page = $this->dom;
524 $allElements = $page->getElementsByTagName('*');
525 /**
526 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
527 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
528 *
529 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
530 * TODO: Shouldn't this be a reverse traversal?
531 **/
532 $node = null;
533 $nodesToScore = array();
534 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
535 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
536 //$node = $targetList->item($nodeIndex);
537 $tagName = strtoupper($node->tagName);
538 /* Remove unlikely candidates */
539 if ($stripUnlikelyCandidates) {
540 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
541 if (
542 preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
543 !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
544 $tagName != 'BODY'
545 )
546 {
547 $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
548 //$nodesToRemove[] = $node;
549 $node->parentNode->removeChild($node);
550 $nodeIndex--;
551 continue;
552 }
553 }
554
555 if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
556 $nodesToScore[] = $node;
557 }
558
559 /* Turn all divs that don't have children block level elements into p's */
560 if ($tagName == 'DIV') {
561 if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
562 //$this->dbg('Altering div to p');
563 $newNode = $this->dom->createElement('p');
564 try {
565 $newNode->innerHTML = $node->innerHTML;
566 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
567 $node->parentNode->replaceChild($newNode, $node);
568 $nodeIndex--;
569 $nodesToScore[] = $node; // or $newNode?
570 }
571 catch(Exception $e) {
572 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
573 }
574 }
575 else
576 {
577 /* EXPERIMENTAL */
578 // TODO: change these p elements back to text nodes after processing
579 for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
580 $childNode = $node->childNodes->item($i);
581 if ($childNode->nodeType == 3) { // XML_TEXT_NODE
582 //$this->dbg('replacing text node with a p tag with the same content.');
583 $p = $this->dom->createElement('p');
584 $p->innerHTML = $childNode->nodeValue;
585 $p->setAttribute('style', 'display: inline;');
586 $p->setAttribute('class', 'readability-styled');
587 $childNode->parentNode->replaceChild($p, $childNode);
588 }
589 }
590 }
591 }
592 }
593
594 /**
595 * Loop through all paragraphs, and assign a score to them based on how content-y they look.
596 * Then add their score to their parent node.
597 *
598 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
599 **/
600 $candidates = array();
601 for ($pt=0; $pt < count($nodesToScore); $pt++) {
602 $parentNode = $nodesToScore[$pt]->parentNode;
603 // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
604 $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
605 $innerText = $this->getInnerText($nodesToScore[$pt]);
606
607 if (!$parentNode || !isset($parentNode->tagName)) {
608 continue;
609 }
610
611 /* If this paragraph is less than 25 characters, don't even count it. */
612 if(strlen($innerText) < 25) {
613 continue;
614 }
615
616 /* Initialize readability data for the parent. */
617 if (!$parentNode->hasAttribute('readability'))
618 {
619 $this->initializeNode($parentNode);
620 $candidates[] = $parentNode;
621 }
622
623 /* Initialize readability data for the grandparent. */
624 if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
625 {
626 $this->initializeNode($grandParentNode);
627 $candidates[] = $grandParentNode;
628 }
629
630 $contentScore = 0;
631
632 /* Add a point for the paragraph itself as a base. */
633 $contentScore++;
634
635 /* Add points for any commas within this paragraph */
636 $contentScore += count(explode(',', $innerText));
637
638 /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
639 $contentScore += min(floor(strlen($innerText) / 100), 3);
640
641 /* Add the score to the parent. The grandparent gets half. */
642 $parentNode->getAttributeNode('readability')->value += $contentScore;
643
644 if ($grandParentNode) {
645 $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
646 }
647 }
648
649 /**
650 * After we've calculated scores, loop through all of the possible candidate nodes we found
651 * and find the one with the highest score.
652 **/
653 $topCandidate = null;
654 for ($c=0, $cl=count($candidates); $c < $cl; $c++)
655 {
656 /**
657 * Scale the final candidates score based on link density. Good content should have a
658 * relatively small link density (5% or less) and be mostly unaffected by this operation.
659 **/
660 $readability = $candidates[$c]->getAttributeNode('readability');
661 $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
662
663 $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
664
665 if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
666 $topCandidate = $candidates[$c];
667 }
668 }
669
670 /**
671 * If we still have no top candidate, just use the body as a last resort.
672 * We also have to copy the body node so it is something we can modify.
673 **/
674 if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
675 {
676 $topCandidate = $this->dom->createElement('div');
677 if ($page instanceof DOMDocument) {
678 if (!isset($page->documentElement)) {
679 // we don't have a body either? what a mess! :)
680 } else {
681 $topCandidate->innerHTML = $page->documentElement->innerHTML;
682 $page->documentElement->innerHTML = '';
683 $this->reinitBody();
684 $page->documentElement->appendChild($topCandidate);
685 }
686 } else {
687 $topCandidate->innerHTML = $page->innerHTML;
688 $page->innerHTML = '';
689 $page->appendChild($topCandidate);
690 }
691 $this->initializeNode($topCandidate);
692 }
693
694 /**
695 * Now that we have the top candidate, look through its siblings for content that might also be related.
696 * Things like preambles, content split by ads that we removed, etc.
697 **/
698 $articleContent = $this->dom->createElement('div');
699 $articleContent->setAttribute('id', 'readability-content');
700 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
701 $siblingNodes = @$topCandidate->parentNode->childNodes;
702 if (!isset($siblingNodes)) {
703 $siblingNodes = new stdClass;
704 $siblingNodes->length = 0;
705 }
706
707 for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
708 {
709 $siblingNode = $siblingNodes->item($s);
710 $append = false;
711
712 $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
713
714 //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
715
716 if ($siblingNode === $topCandidate)
717 // or if ($siblingNode->isSameNode($topCandidate))
718 {
719 $append = true;
720 }
721
722 $contentBonus = 0;
723 /* Give a bonus if sibling nodes and top candidates have the example same classname */
724 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
725 $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
726 }
727
728 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
729 {
730 $append = true;
731 }
732
733 if (strtoupper($siblingNode->nodeName) == 'P') {
734 $linkDensity = $this->getLinkDensity($siblingNode);
735 $nodeContent = $this->getInnerText($siblingNode);
736 $nodeLength = strlen($nodeContent);
737
738 if ($nodeLength > 80 && $linkDensity < 0.25)
739 {
740 $append = true;
741 }
742 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
743 {
744 $append = true;
745 }
746 }
747
748 if ($append)
749 {
750 $this->dbg('Appending node: ' . $siblingNode->nodeName);
751
752 $nodeToAppend = null;
753 $sibNodeName = strtoupper($siblingNode->nodeName);
754 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
755 /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
756
757 $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
758 $nodeToAppend = $this->dom->createElement('div');
759 try {
760 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
761 $nodeToAppend->innerHTML = $siblingNode->innerHTML;
762 }
763 catch(Exception $e)
764 {
765 $this->dbg('Could not alter siblingNode to div, reverting back to original.');
766 $nodeToAppend = $siblingNode;
767 $s--;
768 $sl--;
769 }
770 } else {
771 $nodeToAppend = $siblingNode;
772 $s--;
773 $sl--;
774 }
775
776 /* To ensure a node does not interfere with readability styles, remove its classnames */
777 $nodeToAppend->removeAttribute('class');
778
779 /* Append sibling and subtract from our list because it removes the node when you append to another node */
780 $articleContent->appendChild($nodeToAppend);
781 }
782 }
783
784 /**
785 * So we have all of the content that we need. Now we clean it up for presentation.
786 **/
787 $this->prepArticle($articleContent);
788
789 /**
790 * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
791 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
792 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
793 * finding the -right- content.
794 **/
795 if (strlen($this->getInnerText($articleContent, false)) < 250)
796 {
797 // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
798 // in the meantime, we check and create an empty element if it's not there.
799 $this->reinitBody();
800
801 if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
802 $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
803 return $this->grabArticle($this->body);
804 }
805 else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
806 $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
807 return $this->grabArticle($this->body);
808 }
809 else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
810 $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
811 return $this->grabArticle($this->body);
812 }
813 else {
814 return false;
815 }
816 }
817 return $articleContent;
818 }
819
820 /**
821 * Remove script tags from document
822 *
823 * @param DOMElement
824 * @return void
825 */
826 public function removeScripts($doc) {
827 $scripts = $doc->getElementsByTagName('script');
828 for($i = $scripts->length-1; $i >= 0; $i--)
829 {
830 $scripts->item($i)->parentNode->removeChild($scripts->item($i));
831 }
832 }
833
834 /**
835 * Get the inner text of a node.
836 * This also strips out any excess whitespace to be found.
837 *
838 * @param DOMElement $
839 * @param boolean $normalizeSpaces (default: true)
840 * @return string
841 **/
842 public function getInnerText($e, $normalizeSpaces=true) {
843 $textContent = '';
844
845 if (!isset($e->textContent) || $e->textContent == '') {
846 return '';
847 }
848
849 $textContent = trim($e->textContent);
850
851 if ($normalizeSpaces) {
852 return preg_replace($this->regexps['normalize'], ' ', $textContent);
853 } else {
854 return $textContent;
855 }
856 }
857
858 /**
859 * Get the number of times a string $s appears in the node $e.
860 *
861 * @param DOMElement $e
862 * @param string - what to count. Default is ","
863 * @return number (integer)
864 **/
865 public function getCharCount($e, $s=',') {
866 return substr_count($this->getInnerText($e), $s);
867 }
868
869 /**
870 * Remove the style attribute on every $e and under.
871 *
872 * @param DOMElement $e
873 * @return void
874 */
875 public function cleanStyles($e) {
876 if (!is_object($e)) return;
877 $elems = $e->getElementsByTagName('*');
878 foreach ($elems as $elem) {
879 $elem->removeAttribute('style');
880 }
881 }
882
883 /**
884 * Get the density of links as a percentage of the content
885 * This is the amount of text that is inside a link divided by the total text in the node.
886 *
887 * @param DOMElement $e
888 * @return number (float)
889 */
890 public function getLinkDensity($e) {
891 $links = $e->getElementsByTagName('a');
892 $textLength = strlen($this->getInnerText($e));
893 $linkLength = 0;
894 for ($i=0, $il=$links->length; $i < $il; $i++)
895 {
896 $linkLength += strlen($this->getInnerText($links->item($i)));
897 }
898 if ($textLength > 0) {
899 return $linkLength / $textLength;
900 } else {
901 return 0;
902 }
903 }
904
905 /**
906 * Get an elements class/id weight. Uses regular expressions to tell if this
907 * element looks good or bad.
908 *
909 * @param DOMElement $e
910 * @return number (Integer)
911 */
912 public function getClassWeight($e) {
913 if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
914 return 0;
915 }
916
917 $weight = 0;
918
919 /* Look for a special classname */
920 if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
921 {
922 if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
923 $weight -= 25;
924 }
925 if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
926 $weight += 25;
927 }
928 }
929
930 /* Look for a special ID */
931 if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
932 {
933 if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
934 $weight -= 25;
935 }
936 if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
937 $weight += 25;
938 }
939 }
940 return $weight;
941 }
942
943 /**
944 * Remove extraneous break tags from a node.
945 *
946 * @param DOMElement $node
947 * @return void
948 */
949 public function killBreaks($node) {
950 $html = $node->innerHTML;
951 $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
952 $node->innerHTML = $html;
953 }
954
955 /**
956 * Clean a node of all elements of type "tag".
957 * (Unless it's a youtube/vimeo video. People love movies.)
958 *
959 * Updated 2012-09-18 to preserve youtube/vimeo iframes
960 *
961 * @param DOMElement $e
962 * @param string $tag
963 * @return void
964 */
965 public function clean($e, $tag) {
966 $targetList = $e->getElementsByTagName($tag);
967 $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
968
969 for ($y=$targetList->length-1; $y >= 0; $y--) {
970 /* Allow youtube and vimeo videos through as people usually want to see those. */
971 if ($isEmbed) {
972 $attributeValues = '';
973 for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
974 $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
975 }
976
977 /* First, check the elements attributes to see if any of them contain youtube or vimeo */
978 if (preg_match($this->regexps['video'], $attributeValues)) {
979 continue;
980 }
981
982 /* Then check the elements inside this element for the same. */
983 if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
984 continue;
985 }
986 }
987 $targetList->item($y)->parentNode->removeChild($targetList->item($y));
988 }
989 }
990
991 /**
992 * Clean an element of all tags of type "tag" if they look fishy.
993 * "Fishy" is an algorithm based on content length, classnames,
994 * link density, number of images & embeds, etc.
995 *
996 * @param DOMElement $e
997 * @param string $tag
998 * @return void
999 */
1000 public function cleanConditionally($e, $tag) {
1001 if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
1002 return;
1003 }
1004
1005 $tagsList = $e->getElementsByTagName($tag);
1006 $curTagsLength = $tagsList->length;
1007
1008 /**
1009 * Gather counts for other typical elements embedded within.
1010 * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
1011 *
1012 * TODO: Consider taking into account original contentScore here.
1013 */
1014 for ($i=$curTagsLength-1; $i >= 0; $i--) {
1015 $weight = $this->getClassWeight($tagsList->item($i));
1016 $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
1017
1018 $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
1019
1020 if ($weight + $contentScore < 0) {
1021 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1022 }
1023 else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
1024 /**
1025 * If there are not very many commas, and the number of
1026 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
1027 **/
1028 $p = $tagsList->item($i)->getElementsByTagName('p')->length;
1029 $img = $tagsList->item($i)->getElementsByTagName('img')->length;
1030 $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
1031 $input = $tagsList->item($i)->getElementsByTagName('input')->length;
1032 $a = $tagsList->item($i)->getElementsByTagName('a')->length;
1033
1034 $embedCount = 0;
1035 $embeds = $tagsList->item($i)->getElementsByTagName('embed');
1036 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1037 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1038 $embedCount++;
1039 }
1040 }
1041 $embeds = $tagsList->item($i)->getElementsByTagName('iframe');
1042 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1043 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1044 $embedCount++;
1045 }
1046 }
1047
1048 $linkDensity = $this->getLinkDensity($tagsList->item($i));
1049 $contentLength = strlen($this->getInnerText($tagsList->item($i)));
1050 $toRemove = false;
1051
1052 if ($this->lightClean) {
1053 $this->dbg('Light clean...');
1054 if ( ($img > $p) && ($img > 4) ) {
1055 $this->dbg(' more than 4 images and more image elements than paragraph elements');
1056 $toRemove = true;
1057 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1058 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1059 $toRemove = true;
1060 } else if ( $input > floor($p/3) ) {
1061 $this->dbg(' too many <input> elements');
1062 $toRemove = true;
1063 } else if ($contentLength < 10 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
1064 $this->dbg(' content length less than 10 chars, 0 embeds and either 0 images or more than 2 images');
1065 $toRemove = true;
1066 } else if($weight < 25 && $linkDensity > 0.2) {
1067 $this->dbg(' weight smaller than 25 and link density above 0.2');
1068 $toRemove = true;
1069 } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
1070 $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
1071 $toRemove = true;
1072 } else if($embedCount > 3) {
1073 $this->dbg(' more than 3 embeds');
1074 $toRemove = true;
1075 }
1076 } else {
1077 $this->dbg('Standard clean...');
1078 if ( $img > $p ) {
1079 $this->dbg(' more image elements than paragraph elements');
1080 $toRemove = true;
1081 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1082 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1083 $toRemove = true;
1084 } else if ( $input > floor($p/3) ) {
1085 $this->dbg(' too many <input> elements');
1086 $toRemove = true;
1087 } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
1088 $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
1089 $toRemove = true;
1090 } else if($weight < 25 && $linkDensity > 0.2) {
1091 $this->dbg(' weight smaller than 25 and link density above 0.2');
1092 $toRemove = true;
1093 } else if($weight >= 25 && $linkDensity > 0.5) {
1094 $this->dbg(' weight above 25 but link density greater than 0.5');
1095 $toRemove = true;
1096 } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
1097 $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
1098 $toRemove = true;
1099 }
1100 }
1101
1102 if ($toRemove) {
1103 //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
1104 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1105 }
1106 }
1107 }
1108 }
1109
1110 /**
1111 * Clean out spurious headers from an Element. Checks things like classnames and link density.
1112 *
1113 * @param DOMElement $e
1114 * @return void
1115 */
1116 public function cleanHeaders($e) {
1117 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
1118 $headers = $e->getElementsByTagName('h' . $headerIndex);
1119 for ($i=$headers->length-1; $i >=0; $i--) {
1120 if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
1121 $headers->item($i)->parentNode->removeChild($headers->item($i));
1122 }
1123 }
1124 }
1125 }
1126
1127 public function flagIsActive($flag) {
1128 return ($this->flags & $flag) > 0;
1129 }
1130
1131 public function addFlag($flag) {
1132 $this->flags = $this->flags | $flag;
1133 }
1134
1135 public function removeFlag($flag) {
1136 $this->flags = $this->flags & ~$flag;
1137 }
1138
1139 /**
1140 * Will recreate previously deleted body property
1141 *
1142 * @return void
1143 */
1144 protected function reinitBody() {
1145 if (!isset($this->body->childNodes)) {
1146 $this->body = $this->dom->createElement('body');
1147 $this->body->innerHTML = $this->bodyCache;
1148 }
1149 }
1150
1151}
1152?>
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-13 07:36:03 +0000