]>
Commit | Line | Data |
---|---|---|
1a268ba7 NL |
1 | <?php |
2 | /** | |
3 | * Arc90's Readability ported to PHP for FiveFilters.org | |
4 | * Based on readability.js version 1.7.1 (without multi-page support) | |
5 | * ------------------------------------------------------ | |
6 | * Original URL: http://lab.arc90.com/experiments/readability/js/readability.js | |
7 | * Arc90's project URL: http://lab.arc90.com/experiments/readability/ | |
8 | * JS Source: http://code.google.com/p/arc90labs-readability | |
9 | * Ported by: Keyvan Minoukadeh, http://www.keyvan.net | |
10 | * More information: http://fivefilters.org/content-only/ | |
11 | * License: Apache License, Version 2.0 | |
12 | * Requires: PHP5 | |
13 | * Date: 2011-07-22 | |
14 | * | |
15 | * Differences between the PHP port and the original | |
16 | * ------------------------------------------------------ | |
17 | * Arc90's Readability is designed to run in the browser. It works on the DOM | |
18 | * tree (the parsed HTML) after the page's CSS styles have been applied and | |
19 | * Javascript code executed. This PHP port does not run inside a browser. | |
20 | * We use PHP's ability to parse HTML to build our DOM tree, but we cannot | |
21 | * rely on CSS or Javascript support. As such, the results will not always | |
22 | * match Arc90's Readability. (For example, if a web page contains CSS style | |
23 | * rules or Javascript code which hide certain HTML elements from display, | |
24 | * Arc90's Readability will dismiss those from consideration but our PHP port, | |
25 | * unable to understand CSS or Javascript, will not know any better.) | |
26 | * | |
27 | * Another significant difference is that the aim of Arc90's Readability is | |
28 | * to re-present the main content block of a given web page so users can | |
29 | * read it more easily in their browsers. Correct identification, clean up, | |
30 | * and separation of the content block is only a part of this process. | |
31 | * This PHP port is only concerned with this part, it does not include code | |
32 | * that relates to presentation in the browser - Arc90 already do | |
33 | * that extremely well, and for PDF output there's FiveFilters.org's | |
34 | * PDF Newspaper: http://fivefilters.org/pdf-newspaper/. | |
35 | * | |
36 | * Finally, this class contains methods that might be useful for developers | |
37 | * working on HTML document fragments. So without deviating too much from | |
38 | * the original code (which I don't want to do because it makes debugging | |
39 | * and updating more difficult), I've tried to make it a little more | |
40 | * developer friendly. You should be able to use the methods here on | |
41 | * existing DOMElement objects without passing an entire HTML document to | |
42 | * be parsed. | |
43 | */ | |
44 | ||
45 | // This class allows us to do JavaScript like assignements to innerHTML | |
46 | require_once(dirname(__FILE__).'/JSLikeHTMLElement.php'); | |
47 | ||
48 | // Alternative usage (for testing only!) | |
49 | // uncomment the lines below and call Readability.php in your browser | |
50 | // passing it the URL of the page you'd like content from, e.g.: | |
51 | // Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php | |
52 | ||
53 | /* | |
54 | if (!isset($_GET['url']) || $_GET['url'] == '') { | |
55 | die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html'); | |
56 | } | |
57 | $url = $_GET['url']; | |
58 | if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url; | |
59 | $html = file_get_contents($url); | |
60 | $r = new Readability($html, $url); | |
61 | $r->init(); | |
62 | echo $r->articleContent->innerHTML; | |
63 | */ | |
64 | ||
65 | ||
66 | class Readability | |
67 | { | |
68 | /* constants */ | |
69 | const FLAG_STRIP_UNLIKELYS = 1; | |
70 | const FLAG_WEIGHT_CLASSES = 2; | |
71 | const FLAG_CLEAN_CONDITIONALLY = 4; | |
72 | ||
73 | public $version = '1.7.1-without-multi-page'; | |
74 | public $convertLinksToFootnotes = false; | |
75 | public $revertForcedParagraphElements = true; | |
76 | public $articleTitle; | |
77 | public $articleContent; | |
78 | public $dom; | |
79 | public $url = null; // optional - URL where HTML was retrieved | |
80 | public $debug = false; | |
81 | protected $body = null; // | |
82 | protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later | |
ae957169 | 83 | protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. |
1a268ba7 NL |
84 | protected $success = false; // indicates whether we were able to extract or not |
85 | ||
86 | /** | |
87 | * All of the regular expressions in use within readability. | |
88 | * Defined up here so we don't instantiate them repeatedly in loops. | |
89 | **/ | |
90 | public $regexps = array( | |
91 | 'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i', | |
92 | 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', | |
4ddbd267 | 93 | 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i', |
1a268ba7 NL |
94 | 'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', |
95 | 'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i', | |
96 | 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i', | |
97 | 'replaceFonts' => '/<(\/?)font[^>]*>/i', | |
98 | // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim() | |
99 | 'normalize' => '/\s{2,}/', | |
100 | 'killBreaks' => '/(<br\s*\/?>(\s| ?)*){1,}/', | |
101 | 'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i', | |
102 | 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i' | |
103 | ); | |
104 | ||
105 | /** | |
106 | * Create instance of Readability | |
107 | * @param string UTF-8 encoded string | |
108 | * @param string (optional) URL associated with HTML (used for footnotes) | |
4ddbd267 | 109 | */ |
1a268ba7 NL |
110 | function __construct($html, $url=null) |
111 | { | |
112 | /* Turn all double br's into p's */ | |
113 | $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html); | |
114 | $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); | |
115 | $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); | |
116 | $this->dom = new DOMDocument(); | |
117 | $this->dom->preserveWhiteSpace = false; | |
118 | $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); | |
119 | if (trim($html) == '') $html = '<html></html>'; | |
120 | @$this->dom->loadHTML($html); | |
121 | $this->url = $url; | |
122 | } | |
123 | ||
124 | /** | |
125 | * Get article title element | |
126 | * @return DOMElement | |
127 | */ | |
128 | public function getTitle() { | |
129 | return $this->articleTitle; | |
130 | } | |
131 | ||
132 | /** | |
133 | * Get article content element | |
134 | * @return DOMElement | |
135 | */ | |
136 | public function getContent() { | |
137 | return $this->articleContent; | |
138 | } | |
139 | ||
140 | /** | |
141 | * Runs readability. | |
142 | * | |
143 | * Workflow: | |
144 | * 1. Prep the document by removing script tags, css, etc. | |
145 | * 2. Build readability's DOM tree. | |
146 | * 3. Grab the article content from the current dom tree. | |
147 | * 4. Replace the current DOM tree with the new one. | |
148 | * 5. Read peacefully. | |
149 | * | |
150 | * @return boolean true if we found content, false otherwise | |
151 | **/ | |
152 | public function init() | |
153 | { | |
154 | if (!isset($this->dom->documentElement)) return false; | |
155 | $this->removeScripts($this->dom); | |
156 | //die($this->getInnerHTML($this->dom->documentElement)); | |
157 | ||
158 | // Assume successful outcome | |
159 | $this->success = true; | |
160 | ||
161 | $bodyElems = $this->dom->getElementsByTagName('body'); | |
162 | if ($bodyElems->length > 0) { | |
163 | if ($this->bodyCache == null) { | |
164 | $this->bodyCache = $bodyElems->item(0)->innerHTML; | |
165 | } | |
166 | if ($this->body == null) { | |
167 | $this->body = $bodyElems->item(0); | |
168 | } | |
169 | } | |
170 | ||
171 | $this->prepDocument(); | |
172 | ||
173 | //die($this->dom->documentElement->parentNode->nodeType); | |
174 | //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); | |
175 | //die($this->getInnerHTML($this->dom->documentElement)); | |
176 | ||
177 | /* Build readability's DOM tree */ | |
178 | $overlay = $this->dom->createElement('div'); | |
179 | $innerDiv = $this->dom->createElement('div'); | |
180 | $articleTitle = $this->getArticleTitle(); | |
181 | $articleContent = $this->grabArticle(); | |
182 | ||
183 | if (!$articleContent) { | |
184 | $this->success = false; | |
185 | $articleContent = $this->dom->createElement('div'); | |
186 | $articleContent->setAttribute('id', 'readability-content'); | |
187 | $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>'; | |
4ddbd267 | 188 | return $this->success; |
1a268ba7 NL |
189 | } |
190 | ||
191 | $overlay->setAttribute('id', 'readOverlay'); | |
192 | $innerDiv->setAttribute('id', 'readInner'); | |
193 | ||
194 | /* Glue the structure of our document together. */ | |
195 | $innerDiv->appendChild($articleTitle); | |
196 | $innerDiv->appendChild($articleContent); | |
197 | $overlay->appendChild($innerDiv); | |
198 | ||
199 | /* Clear the old HTML, insert the new content. */ | |
200 | $this->body->innerHTML = ''; | |
201 | $this->body->appendChild($overlay); | |
202 | //document.body.insertBefore(overlay, document.body.firstChild); | |
203 | $this->body->removeAttribute('style'); | |
204 | ||
205 | $this->postProcessContent($articleContent); | |
206 | ||
207 | // Set title and content instance variables | |
208 | $this->articleTitle = $articleTitle; | |
209 | $this->articleContent = $articleContent; | |
210 | ||
211 | return $this->success; | |
212 | } | |
213 | ||
214 | /** | |
215 | * Debug | |
216 | */ | |
217 | protected function dbg($msg) { | |
218 | if ($this->debug) echo '* ',$msg, '<br />', "\n"; | |
219 | } | |
220 | ||
221 | /** | |
222 | * Run any post-process modifications to article content as necessary. | |
223 | * | |
224 | * @param DOMElement | |
225 | * @return void | |
226 | */ | |
227 | public function postProcessContent($articleContent) { | |
228 | if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { | |
229 | $this->addFootnotes($articleContent); | |
230 | } | |
231 | } | |
232 | ||
233 | /** | |
234 | * Get the article title as an H1. | |
235 | * | |
236 | * @return DOMElement | |
237 | */ | |
238 | protected function getArticleTitle() { | |
239 | $curTitle = ''; | |
240 | $origTitle = ''; | |
241 | ||
242 | try { | |
243 | $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0)); | |
244 | } catch(Exception $e) {} | |
245 | ||
246 | if (preg_match('/ [\|\-] /', $curTitle)) | |
247 | { | |
248 | $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle); | |
249 | ||
250 | if (count(explode(' ', $curTitle)) < 3) { | |
251 | $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle); | |
252 | } | |
253 | } | |
254 | else if (strpos($curTitle, ': ') !== false) | |
255 | { | |
256 | $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle); | |
257 | ||
258 | if (count(explode(' ', $curTitle)) < 3) { | |
259 | $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle); | |
260 | } | |
261 | } | |
262 | else if(strlen($curTitle) > 150 || strlen($curTitle) < 15) | |
263 | { | |
264 | $hOnes = $this->dom->getElementsByTagName('h1'); | |
265 | if($hOnes->length == 1) | |
266 | { | |
267 | $curTitle = $this->getInnerText($hOnes->item(0)); | |
268 | } | |
269 | } | |
270 | ||
271 | $curTitle = trim($curTitle); | |
272 | ||
273 | if (count(explode(' ', $curTitle)) <= 4) { | |
274 | $curTitle = $origTitle; | |
275 | } | |
276 | ||
277 | $articleTitle = $this->dom->createElement('h1'); | |
278 | $articleTitle->innerHTML = $curTitle; | |
279 | ||
280 | return $articleTitle; | |
281 | } | |
282 | ||
283 | /** | |
284 | * Prepare the HTML document for readability to scrape it. | |
285 | * This includes things like stripping javascript, CSS, and handling terrible markup. | |
286 | * | |
287 | * @return void | |
288 | **/ | |
289 | protected function prepDocument() { | |
290 | /** | |
291 | * In some cases a body element can't be found (if the HTML is totally hosed for example) | |
292 | * so we create a new body node and append it to the document. | |
293 | */ | |
294 | if ($this->body == null) | |
295 | { | |
296 | $this->body = $this->dom->createElement('body'); | |
297 | $this->dom->documentElement->appendChild($this->body); | |
298 | } | |
299 | $this->body->setAttribute('id', 'readabilityBody'); | |
300 | ||
301 | /* Remove all style tags in head */ | |
302 | $styleTags = $this->dom->getElementsByTagName('style'); | |
303 | for ($i = $styleTags->length-1; $i >= 0; $i--) | |
304 | { | |
305 | $styleTags->item($i)->parentNode->removeChild($styleTags->item($i)); | |
306 | } | |
307 | ||
308 | /* Turn all double br's into p's */ | |
309 | /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */ | |
310 | //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>'); | |
311 | // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. | |
312 | // Manipulating innerHTML as it's done in JS is not possible in PHP. | |
313 | } | |
314 | ||
315 | /** | |
316 | * For easier reading, convert this document to have footnotes at the bottom rather than inline links. | |
317 | * @see http://www.roughtype.com/archives/2010/05/experiments_in.php | |
318 | * | |
319 | * @return void | |
320 | **/ | |
321 | public function addFootnotes($articleContent) { | |
322 | $footnotesWrapper = $this->dom->createElement('div'); | |
323 | $footnotesWrapper->setAttribute('id', 'readability-footnotes'); | |
324 | $footnotesWrapper->innerHTML = '<h3>References</h3>'; | |
325 | ||
326 | $articleFootnotes = $this->dom->createElement('ol'); | |
327 | $articleFootnotes->setAttribute('id', 'readability-footnotes-list'); | |
328 | $footnotesWrapper->appendChild($articleFootnotes); | |
329 | ||
330 | $articleLinks = $articleContent->getElementsByTagName('a'); | |
331 | ||
332 | $linkCount = 0; | |
333 | for ($i = 0; $i < $articleLinks->length; $i++) | |
334 | { | |
335 | $articleLink = $articleLinks->item($i); | |
336 | $footnoteLink = $articleLink->cloneNode(true); | |
337 | $refLink = $this->dom->createElement('a'); | |
338 | $footnote = $this->dom->createElement('li'); | |
339 | $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST); | |
340 | if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST); | |
341 | //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host, | |
342 | $linkText = $this->getInnerText($articleLink); | |
343 | ||
344 | if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) { | |
345 | continue; | |
346 | } | |
347 | ||
348 | $linkCount++; | |
349 | ||
350 | /** Add a superscript reference after the article link */ | |
351 | $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount); | |
352 | $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>'; | |
353 | $refLink->setAttribute('class', 'readability-DoNotFootnote'); | |
354 | $refLink->setAttribute('style', 'color: inherit;'); | |
355 | ||
356 | //TODO: does this work or should we use DOMNode.isSameNode()? | |
357 | if ($articleLink->parentNode->lastChild == $articleLink) { | |
358 | $articleLink->parentNode->appendChild($refLink); | |
359 | } else { | |
360 | $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling); | |
361 | } | |
362 | ||
363 | $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;'); | |
364 | $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount); | |
365 | ||
366 | $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> '; | |
367 | ||
368 | $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText); | |
369 | $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount); | |
370 | ||
371 | $footnote->appendChild($footnoteLink); | |
372 | if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>'; | |
373 | ||
374 | $articleFootnotes->appendChild($footnote); | |
375 | } | |
376 | ||
377 | if ($linkCount > 0) { | |
378 | $articleContent->appendChild($footnotesWrapper); | |
379 | } | |
380 | } | |
381 | ||
382 | /** | |
383 | * Reverts P elements with class 'readability-styled' | |
384 | * to text nodes - which is what they were before. | |
385 | * | |
386 | * @param DOMElement | |
387 | * @return void | |
388 | */ | |
389 | function revertReadabilityStyledElements($articleContent) { | |
390 | $xpath = new DOMXPath($articleContent->ownerDocument); | |
391 | $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent); | |
392 | //$elems = $articleContent->getElementsByTagName('p'); | |
393 | for ($i = $elems->length-1; $i >= 0; $i--) { | |
394 | $e = $elems->item($i); | |
395 | $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); | |
396 | //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') { | |
397 | // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e); | |
398 | //} | |
399 | } | |
400 | } | |
401 | ||
402 | /** | |
403 | * Prepare the article node for display. Clean out any inline styles, | |
404 | * iframes, forms, strip extraneous <p> tags, etc. | |
405 | * | |
406 | * @param DOMElement | |
407 | * @return void | |
408 | */ | |
409 | function prepArticle($articleContent) { | |
410 | $this->cleanStyles($articleContent); | |
411 | $this->killBreaks($articleContent); | |
412 | if ($this->revertForcedParagraphElements) { | |
413 | $this->revertReadabilityStyledElements($articleContent); | |
414 | } | |
415 | ||
416 | /* Clean out junk from the article content */ | |
417 | $this->cleanConditionally($articleContent, 'form'); | |
418 | $this->clean($articleContent, 'object'); | |
419 | $this->clean($articleContent, 'h1'); | |
420 | ||
421 | /** | |
422 | * If there is only one h2, they are probably using it | |
423 | * as a header and not a subheader, so remove it since we already have a header. | |
424 | ***/ | |
425 | if ($articleContent->getElementsByTagName('h2')->length == 1) { | |
426 | $this->clean($articleContent, 'h2'); | |
427 | } | |
428 | $this->clean($articleContent, 'iframe'); | |
429 | ||
430 | $this->cleanHeaders($articleContent); | |
431 | ||
432 | /* Do these last as the previous stuff may have removed junk that will affect these */ | |
433 | $this->cleanConditionally($articleContent, 'table'); | |
434 | $this->cleanConditionally($articleContent, 'ul'); | |
435 | $this->cleanConditionally($articleContent, 'div'); | |
436 | ||
437 | /* Remove extra paragraphs */ | |
438 | $articleParagraphs = $articleContent->getElementsByTagName('p'); | |
439 | for ($i = $articleParagraphs->length-1; $i >= 0; $i--) | |
440 | { | |
441 | $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length; | |
442 | $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length; | |
443 | $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length; | |
444 | ||
445 | if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '') | |
446 | { | |
447 | $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i)); | |
448 | } | |
449 | } | |
450 | ||
451 | try { | |
452 | $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML); | |
453 | //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p'); | |
454 | } | |
455 | catch (Exception $e) { | |
456 | $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e); | |
457 | } | |
458 | } | |
459 | ||
460 | /** | |
461 | * Initialize a node with the readability object. Also checks the | |
462 | * className/id for special names to add to its score. | |
463 | * | |
464 | * @param Element | |
465 | * @return void | |
466 | **/ | |
467 | protected function initializeNode($node) { | |
468 | $readability = $this->dom->createAttribute('readability'); | |
469 | $readability->value = 0; // this is our contentScore | |
470 | $node->setAttributeNode($readability); | |
471 | ||
472 | switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case | |
473 | case 'DIV': | |
474 | $readability->value += 5; | |
475 | break; | |
476 | ||
477 | case 'PRE': | |
478 | case 'TD': | |
479 | case 'BLOCKQUOTE': | |
480 | $readability->value += 3; | |
481 | break; | |
482 | ||
483 | case 'ADDRESS': | |
484 | case 'OL': | |
485 | case 'UL': | |
486 | case 'DL': | |
487 | case 'DD': | |
488 | case 'DT': | |
489 | case 'LI': | |
490 | case 'FORM': | |
491 | $readability->value -= 3; | |
492 | break; | |
493 | ||
494 | case 'H1': | |
495 | case 'H2': | |
496 | case 'H3': | |
497 | case 'H4': | |
498 | case 'H5': | |
499 | case 'H6': | |
500 | case 'TH': | |
501 | $readability->value -= 5; | |
502 | break; | |
503 | } | |
504 | $readability->value += $this->getClassWeight($node); | |
505 | } | |
506 | ||
507 | /*** | |
508 | * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is | |
509 | * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. | |
510 | * | |
511 | * @return DOMElement | |
512 | **/ | |
513 | protected function grabArticle($page=null) { | |
514 | $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS); | |
515 | if (!$page) $page = $this->dom; | |
516 | $allElements = $page->getElementsByTagName('*'); | |
517 | /** | |
518 | * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs | |
519 | * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.) | |
520 | * | |
521 | * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 | |
522 | * TODO: Shouldn't this be a reverse traversal? | |
523 | **/ | |
524 | $node = null; | |
525 | $nodesToScore = array(); | |
526 | for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) { | |
527 | //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) { | |
528 | //$node = $targetList->item($nodeIndex); | |
529 | $tagName = strtoupper($node->tagName); | |
530 | /* Remove unlikely candidates */ | |
531 | if ($stripUnlikelyCandidates) { | |
532 | $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id'); | |
533 | if ( | |
534 | preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) && | |
535 | !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) && | |
536 | $tagName != 'BODY' | |
537 | ) | |
538 | { | |
539 | $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString); | |
540 | //$nodesToRemove[] = $node; | |
541 | $node->parentNode->removeChild($node); | |
542 | $nodeIndex--; | |
543 | continue; | |
544 | } | |
545 | } | |
546 | ||
547 | if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') { | |
548 | $nodesToScore[] = $node; | |
549 | } | |
550 | ||
551 | /* Turn all divs that don't have children block level elements into p's */ | |
552 | if ($tagName == 'DIV') { | |
553 | if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) { | |
554 | //$this->dbg('Altering div to p'); | |
555 | $newNode = $this->dom->createElement('p'); | |
556 | try { | |
557 | $newNode->innerHTML = $node->innerHTML; | |
558 | //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node); | |
559 | $node->parentNode->replaceChild($newNode, $node); | |
560 | $nodeIndex--; | |
561 | $nodesToScore[] = $node; // or $newNode? | |
562 | } | |
563 | catch(Exception $e) { | |
564 | $this->dbg('Could not alter div to p, reverting back to div.: ' . $e); | |
565 | } | |
566 | } | |
567 | else | |
568 | { | |
569 | // EXPERIMENTAL | |
570 | // TODO: change these p elements back to text nodes after processing | |
571 | for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) { | |
572 | $childNode = $node->childNodes->item($i); | |
573 | if ($childNode->nodeType == 3) { // XML_TEXT_NODE | |
574 | //$this->dbg('replacing text node with a p tag with the same content.'); | |
575 | $p = $this->dom->createElement('p'); | |
576 | $p->innerHTML = $childNode->nodeValue; | |
577 | $p->setAttribute('style', 'display: inline;'); | |
578 | $p->setAttribute('class', 'readability-styled'); | |
579 | $childNode->parentNode->replaceChild($p, $childNode); | |
580 | } | |
581 | } | |
582 | } | |
583 | } | |
584 | } | |
585 | ||
586 | /** | |
587 | * Loop through all paragraphs, and assign a score to them based on how content-y they look. | |
588 | * Then add their score to their parent node. | |
589 | * | |
590 | * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. | |
591 | **/ | |
592 | $candidates = array(); | |
593 | for ($pt=0; $pt < count($nodesToScore); $pt++) { | |
594 | $parentNode = $nodesToScore[$pt]->parentNode; | |
595 | // $grandParentNode = $parentNode ? $parentNode->parentNode : null; | |
596 | $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null); | |
597 | $innerText = $this->getInnerText($nodesToScore[$pt]); | |
598 | ||
599 | if (!$parentNode || !isset($parentNode->tagName)) { | |
600 | continue; | |
601 | } | |
602 | ||
603 | /* If this paragraph is less than 25 characters, don't even count it. */ | |
604 | if(strlen($innerText) < 25) { | |
605 | continue; | |
606 | } | |
607 | ||
608 | /* Initialize readability data for the parent. */ | |
609 | if (!$parentNode->hasAttribute('readability')) | |
610 | { | |
611 | $this->initializeNode($parentNode); | |
612 | $candidates[] = $parentNode; | |
613 | } | |
614 | ||
615 | /* Initialize readability data for the grandparent. */ | |
616 | if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) | |
617 | { | |
618 | $this->initializeNode($grandParentNode); | |
619 | $candidates[] = $grandParentNode; | |
620 | } | |
621 | ||
622 | $contentScore = 0; | |
623 | ||
624 | /* Add a point for the paragraph itself as a base. */ | |
625 | $contentScore++; | |
626 | ||
627 | /* Add points for any commas within this paragraph */ | |
628 | $contentScore += count(explode(',', $innerText)); | |
629 | ||
630 | /* For every 100 characters in this paragraph, add another point. Up to 3 points. */ | |
631 | $contentScore += min(floor(strlen($innerText) / 100), 3); | |
632 | ||
633 | /* Add the score to the parent. The grandparent gets half. */ | |
634 | $parentNode->getAttributeNode('readability')->value += $contentScore; | |
635 | ||
636 | if ($grandParentNode) { | |
637 | $grandParentNode->getAttributeNode('readability')->value += $contentScore/2; | |
638 | } | |
639 | } | |
640 | ||
641 | /** | |
642 | * After we've calculated scores, loop through all of the possible candidate nodes we found | |
643 | * and find the one with the highest score. | |
644 | **/ | |
645 | $topCandidate = null; | |
646 | for ($c=0, $cl=count($candidates); $c < $cl; $c++) | |
647 | { | |
648 | /** | |
649 | * Scale the final candidates score based on link density. Good content should have a | |
650 | * relatively small link density (5% or less) and be mostly unaffected by this operation. | |
651 | **/ | |
652 | $readability = $candidates[$c]->getAttributeNode('readability'); | |
653 | $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c])); | |
654 | ||
655 | $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value); | |
656 | ||
657 | if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) { | |
658 | $topCandidate = $candidates[$c]; | |
659 | } | |
660 | } | |
661 | ||
662 | /** | |
663 | * If we still have no top candidate, just use the body as a last resort. | |
664 | * We also have to copy the body node so it is something we can modify. | |
665 | **/ | |
666 | if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY') | |
667 | { | |
668 | $topCandidate = $this->dom->createElement('div'); | |
669 | if ($page instanceof DOMDocument) { | |
670 | if (!isset($page->documentElement)) { | |
671 | // we don't have a body either? what a mess! :) | |
672 | } else { | |
673 | $topCandidate->innerHTML = $page->documentElement->innerHTML; | |
674 | $page->documentElement->innerHTML = ''; | |
675 | $page->documentElement->appendChild($topCandidate); | |
676 | } | |
677 | } else { | |
678 | $topCandidate->innerHTML = $page->innerHTML; | |
679 | $page->innerHTML = ''; | |
680 | $page->appendChild($topCandidate); | |
681 | } | |
682 | $this->initializeNode($topCandidate); | |
683 | } | |
684 | ||
685 | /** | |
686 | * Now that we have the top candidate, look through its siblings for content that might also be related. | |
687 | * Things like preambles, content split by ads that we removed, etc. | |
688 | **/ | |
689 | $articleContent = $this->dom->createElement('div'); | |
690 | $articleContent->setAttribute('id', 'readability-content'); | |
691 | $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2); | |
692 | $siblingNodes = $topCandidate->parentNode->childNodes; | |
693 | if (!isset($siblingNodes)) { | |
694 | $siblingNodes = new stdClass; | |
695 | $siblingNodes->length = 0; | |
696 | } | |
697 | ||
698 | for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++) | |
699 | { | |
700 | $siblingNode = $siblingNodes->item($s); | |
701 | $append = false; | |
702 | ||
703 | $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : '')); | |
704 | ||
705 | //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown')); | |
706 | ||
707 | if ($siblingNode === $topCandidate) | |
708 | // or if ($siblingNode->isSameNode($topCandidate)) | |
709 | { | |
710 | $append = true; | |
711 | } | |
712 | ||
713 | $contentBonus = 0; | |
714 | /* Give a bonus if sibling nodes and top candidates have the example same classname */ | |
715 | if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') { | |
716 | $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2; | |
717 | } | |
718 | ||
719 | if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) | |
720 | { | |
721 | $append = true; | |
722 | } | |
723 | ||
724 | if (strtoupper($siblingNode->nodeName) == 'P') { | |
725 | $linkDensity = $this->getLinkDensity($siblingNode); | |
726 | $nodeContent = $this->getInnerText($siblingNode); | |
727 | $nodeLength = strlen($nodeContent); | |
728 | ||
729 | if ($nodeLength > 80 && $linkDensity < 0.25) | |
730 | { | |
731 | $append = true; | |
732 | } | |
733 | else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) | |
734 | { | |
735 | $append = true; | |
736 | } | |
737 | } | |
738 | ||
739 | /* Look for a special classname */ | |
740 | if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('class') && $siblingNode->getAttribute('class') != '') | |
741 | { | |
742 | if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('class'))) { | |
743 | $append = true; | |
744 | } | |
745 | } | |
746 | ||
747 | /* Look for a special classname */ | |
748 | if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('id') && $siblingNode->getAttribute('id') != '') | |
749 | { | |
750 | if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('id'))) { | |
751 | $append = true; | |
752 | } | |
753 | } | |
754 | ||
755 | ||
756 | if ($append) | |
757 | { | |
758 | $this->dbg('Appending node: ' . $siblingNode->nodeName); | |
759 | ||
760 | $nodeToAppend = null; | |
761 | $sibNodeName = strtoupper($siblingNode->nodeName); | |
762 | if ($sibNodeName != 'DIV' && $sibNodeName != 'P') { | |
763 | /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */ | |
764 | ||
765 | $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.'); | |
766 | $nodeToAppend = $this->dom->createElement('div'); | |
767 | try { | |
768 | $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id')); | |
769 | $nodeToAppend->innerHTML = $siblingNode->innerHTML; | |
770 | } | |
771 | catch(Exception $e) | |
772 | { | |
773 | $this->dbg('Could not alter siblingNode to div, reverting back to original.'); | |
774 | $nodeToAppend = $siblingNode; | |
775 | $s--; | |
776 | $sl--; | |
777 | } | |
778 | } else { | |
779 | $nodeToAppend = $siblingNode; | |
780 | $s--; | |
781 | $sl--; | |
782 | } | |
783 | ||
784 | /* To ensure a node does not interfere with readability styles, remove its classnames */ | |
785 | $nodeToAppend->removeAttribute('class'); | |
786 | ||
787 | /* Append sibling and subtract from our list because it removes the node when you append to another node */ | |
788 | $articleContent->appendChild($nodeToAppend); | |
789 | } | |
790 | } | |
791 | ||
792 | /** | |
793 | * So we have all of the content that we need. Now we clean it up for presentation. | |
794 | **/ | |
795 | $this->prepArticle($articleContent); | |
796 | ||
797 | /** | |
798 | * Now that we've gone through the full algorithm, check to see if we got any meaningful content. | |
799 | * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher | |
800 | * likelihood of finding the content, and the sieve approach gives us a higher likelihood of | |
801 | * finding the -right- content. | |
802 | **/ | |
803 | if (strlen($this->getInnerText($articleContent, false)) < 250) | |
804 | { | |
805 | // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7 | |
806 | // in the meantime, we check and create an empty element if it's not there. | |
807 | if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body'); | |
808 | $this->body->innerHTML = $this->bodyCache; | |
809 | ||
810 | if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) { | |
811 | $this->removeFlag(self::FLAG_STRIP_UNLIKELYS); | |
812 | return $this->grabArticle($this->body); | |
813 | } | |
814 | else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { | |
815 | $this->removeFlag(self::FLAG_WEIGHT_CLASSES); | |
816 | return $this->grabArticle($this->body); | |
817 | } | |
818 | else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { | |
819 | $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY); | |
820 | return $this->grabArticle($this->body); | |
821 | } | |
822 | else { | |
823 | return false; | |
824 | } | |
825 | } | |
826 | return $articleContent; | |
827 | } | |
828 | ||
829 | /** | |
830 | * Remove script tags from document | |
831 | * | |
832 | * @param DOMElement | |
833 | * @return void | |
834 | */ | |
835 | public function removeScripts($doc) { | |
836 | $scripts = $doc->getElementsByTagName('script'); | |
837 | for($i = $scripts->length-1; $i >= 0; $i--) | |
838 | { | |
839 | $scripts->item($i)->parentNode->removeChild($scripts->item($i)); | |
840 | } | |
841 | } | |
842 | ||
843 | /** | |
844 | * Get the inner text of a node. | |
845 | * This also strips out any excess whitespace to be found. | |
846 | * | |
847 | * @param DOMElement $ | |
848 | * @param boolean $normalizeSpaces (default: true) | |
849 | * @return string | |
850 | **/ | |
851 | public function getInnerText($e, $normalizeSpaces=true) { | |
852 | $textContent = ''; | |
853 | ||
854 | if (!isset($e->textContent) || $e->textContent == '') { | |
855 | return ''; | |
856 | } | |
857 | ||
858 | $textContent = trim($e->textContent); | |
859 | ||
860 | if ($normalizeSpaces) { | |
861 | return preg_replace($this->regexps['normalize'], ' ', $textContent); | |
862 | } else { | |
863 | return $textContent; | |
864 | } | |
865 | } | |
866 | ||
867 | /** | |
868 | * Get the number of times a string $s appears in the node $e. | |
869 | * | |
870 | * @param DOMElement $e | |
871 | * @param string - what to count. Default is "," | |
872 | * @return number (integer) | |
873 | **/ | |
874 | public function getCharCount($e, $s=',') { | |
875 | return substr_count($this->getInnerText($e), $s); | |
876 | } | |
877 | ||
878 | /** | |
879 | * Remove the style attribute on every $e and under. | |
880 | * | |
881 | * @param DOMElement $e | |
882 | * @return void | |
883 | */ | |
884 | public function cleanStyles($e) { | |
885 | if (!is_object($e)) return; | |
886 | $elems = $e->getElementsByTagName('*'); | |
887 | foreach ($elems as $elem) { | |
888 | $elem->removeAttribute('style'); | |
889 | } | |
890 | } | |
891 | ||
892 | /** | |
893 | * Get the density of links as a percentage of the content | |
894 | * This is the amount of text that is inside a link divided by the total text in the node. | |
895 | * | |
896 | * @param DOMElement $e | |
897 | * @return number (float) | |
898 | */ | |
899 | public function getLinkDensity($e) { | |
900 | $links = $e->getElementsByTagName('a'); | |
901 | $textLength = strlen($this->getInnerText($e)); | |
902 | $linkLength = 0; | |
903 | for ($i=0, $il=$links->length; $i < $il; $i++) | |
904 | { | |
905 | $linkLength += strlen($this->getInnerText($links->item($i))); | |
906 | } | |
907 | if ($textLength > 0) { | |
908 | return $linkLength / $textLength; | |
909 | } else { | |
910 | return 0; | |
911 | } | |
912 | } | |
913 | ||
914 | /** | |
915 | * Get an elements class/id weight. Uses regular expressions to tell if this | |
916 | * element looks good or bad. | |
917 | * | |
918 | * @param DOMElement $e | |
919 | * @return number (Integer) | |
920 | */ | |
921 | public function getClassWeight($e) { | |
922 | if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { | |
923 | return 0; | |
924 | } | |
925 | ||
926 | $weight = 0; | |
927 | ||
928 | /* Look for a special classname */ | |
929 | if ($e->hasAttribute('class') && $e->getAttribute('class') != '') | |
930 | { | |
931 | if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) { | |
932 | $weight -= 25; | |
933 | } | |
934 | if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) { | |
935 | $weight += 25; | |
936 | } | |
937 | } | |
938 | ||
939 | /* Look for a special ID */ | |
940 | if ($e->hasAttribute('id') && $e->getAttribute('id') != '') | |
941 | { | |
942 | if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) { | |
943 | $weight -= 25; | |
944 | } | |
945 | if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) { | |
946 | $weight += 25; | |
947 | } | |
948 | } | |
949 | return $weight; | |
950 | } | |
951 | ||
952 | /** | |
953 | * Remove extraneous break tags from a node. | |
954 | * | |
955 | * @param DOMElement $node | |
956 | * @return void | |
957 | */ | |
958 | public function killBreaks($node) { | |
959 | $html = $node->innerHTML; | |
960 | $html = preg_replace($this->regexps['killBreaks'], '<br />', $html); | |
961 | $node->innerHTML = $html; | |
962 | } | |
963 | ||
964 | /** | |
965 | * Clean a node of all elements of type "tag". | |
966 | * (Unless it's a youtube/vimeo video. People love movies.) | |
967 | * | |
968 | * @param DOMElement $e | |
969 | * @param string $tag | |
970 | * @return void | |
971 | */ | |
972 | public function clean($e, $tag) { | |
973 | $targetList = $e->getElementsByTagName($tag); | |
974 | $isEmbed = ($tag == 'object' || $tag == 'embed'); | |
975 | ||
976 | for ($y=$targetList->length-1; $y >= 0; $y--) { | |
977 | /* Allow youtube and vimeo videos through as people usually want to see those. */ | |
978 | if ($isEmbed) { | |
979 | $attributeValues = ''; | |
980 | for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) { | |
981 | $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test) | |
982 | } | |
983 | ||
984 | /* First, check the elements attributes to see if any of them contain youtube or vimeo */ | |
985 | if (preg_match($this->regexps['video'], $attributeValues)) { | |
986 | continue; | |
987 | } | |
988 | ||
989 | /* Then check the elements inside this element for the same. */ | |
990 | if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) { | |
991 | continue; | |
992 | } | |
993 | } | |
994 | $targetList->item($y)->parentNode->removeChild($targetList->item($y)); | |
995 | } | |
996 | } | |
997 | ||
998 | /** | |
999 | * Clean an element of all tags of type "tag" if they look fishy. | |
1000 | * "Fishy" is an algorithm based on content length, classnames, | |
1001 | * link density, number of images & embeds, etc. | |
1002 | * | |
1003 | * @param DOMElement $e | |
1004 | * @param string $tag | |
1005 | * @return void | |
1006 | */ | |
1007 | public function cleanConditionally($e, $tag) { | |
1008 | if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { | |
1009 | return; | |
1010 | } | |
1011 | ||
1012 | $tagsList = $e->getElementsByTagName($tag); | |
1013 | $curTagsLength = $tagsList->length; | |
1014 | ||
1015 | /** | |
1016 | * Gather counts for other typical elements embedded within. | |
1017 | * Traverse backwards so we can remove nodes at the same time without effecting the traversal. | |
1018 | * | |
1019 | * TODO: Consider taking into account original contentScore here. | |
1020 | */ | |
1021 | for ($i=$curTagsLength-1; $i >= 0; $i--) { | |
1022 | $weight = $this->getClassWeight($tagsList->item($i)); | |
1023 | $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0; | |
1024 | ||
1025 | $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : '')); | |
1026 | ||
1027 | if ($weight + $contentScore < 0) { | |
1028 | $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); | |
1029 | } | |
1030 | else if ( $this->getCharCount($tagsList->item($i), ',') < 10) { | |
1031 | /** | |
1032 | * If there are not very many commas, and the number of | |
1033 | * non-paragraph elements is more than paragraphs or other ominous signs, remove the element. | |
1034 | **/ | |
1035 | $p = $tagsList->item($i)->getElementsByTagName('p')->length; | |
1036 | $img = $tagsList->item($i)->getElementsByTagName('img')->length; | |
1037 | $li = $tagsList->item($i)->getElementsByTagName('li')->length-100; | |
1038 | $input = $tagsList->item($i)->getElementsByTagName('input')->length; | |
1039 | ||
1040 | $embedCount = 0; | |
1041 | $embeds = $tagsList->item($i)->getElementsByTagName('embed'); | |
1042 | for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) { | |
1043 | if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) { | |
1044 | $embedCount++; | |
1045 | } | |
1046 | } | |
1047 | ||
1048 | $linkDensity = $this->getLinkDensity($tagsList->item($i)); | |
1049 | $contentLength = strlen($this->getInnerText($tagsList->item($i))); | |
1050 | $toRemove = false; | |
1051 | ||
1052 | if ( $img > $p ) { | |
1053 | $toRemove = true; | |
1054 | } else if ($li > $p && $tag != 'ul' && $tag != 'ol') { | |
1055 | $toRemove = true; | |
1056 | } else if ( $input > floor($p/3) ) { | |
1057 | $toRemove = true; | |
1058 | } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) { | |
1059 | $toRemove = true; | |
1060 | } else if($weight < 25 && $linkDensity > 0.2) { | |
1061 | $toRemove = true; | |
1062 | } else if($weight >= 25 && $linkDensity > 0.5) { | |
1063 | $toRemove = true; | |
1064 | } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) { | |
1065 | $toRemove = true; | |
1066 | } | |
1067 | ||
1068 | if ($toRemove) { | |
1069 | $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); | |
1070 | } | |
1071 | } | |
1072 | } | |
1073 | } | |
1074 | ||
1075 | /** | |
1076 | * Clean out spurious headers from an Element. Checks things like classnames and link density. | |
1077 | * | |
1078 | * @param DOMElement $e | |
1079 | * @return void | |
1080 | */ | |
1081 | public function cleanHeaders($e) { | |
1082 | for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) { | |
1083 | $headers = $e->getElementsByTagName('h' . $headerIndex); | |
1084 | for ($i=$headers->length-1; $i >=0; $i--) { | |
1085 | if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) { | |
1086 | $headers->item($i)->parentNode->removeChild($headers->item($i)); | |
1087 | } | |
1088 | } | |
1089 | } | |
1090 | } | |
1091 | ||
1092 | public function flagIsActive($flag) { | |
1093 | return ($this->flags & $flag) > 0; | |
1094 | } | |
1095 | ||
1096 | public function addFlag($flag) { | |
1097 | $this->flags = $this->flags | $flag; | |
1098 | } | |
1099 | ||
1100 | public function removeFlag($flag) { | |
1101 | $this->flags = $this->flags & ~$flag; | |
1102 | } | |
1103 | } | |
1104 | ?> |