aboutsummaryrefslogtreecommitdiffhomepage
path: root/inc
diff options
context:
space:
mode:
Diffstat (limited to 'inc')
-rw-r--r--inc/Readability.php2165
1 files changed, 1099 insertions, 1066 deletions
diff --git a/inc/Readability.php b/inc/Readability.php
index 2ce90f6d..d28d28f9 100644
--- a/inc/Readability.php
+++ b/inc/Readability.php
@@ -1,7 +1,9 @@
1<?php 1<?php
2/** 2/**
3* Arc90's Readability ported to PHP for FiveFilters.org 3* Arc90's Readability ported to PHP for FiveFilters.org
4* Based on readability.js version 1.7.1 (without multi-page support) 4* Based on readability.js version 1.7.1 (without multi-page support)
5* Updated to allow HTML5 parsing with html5lib
6* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
5* ------------------------------------------------------ 7* ------------------------------------------------------
6* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js 8* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
7* Arc90's project URL: http://lab.arc90.com/experiments/readability/ 9* Arc90's project URL: http://lab.arc90.com/experiments/readability/
@@ -10,35 +12,35 @@
10* More information: http://fivefilters.org/content-only/ 12* More information: http://fivefilters.org/content-only/
11* License: Apache License, Version 2.0 13* License: Apache License, Version 2.0
12* Requires: PHP5 14* Requires: PHP5
13* Date: 2011-07-22 15* Date: 2012-09-19
14* 16*
15* Differences between the PHP port and the original 17* Differences between the PHP port and the original
16* ------------------------------------------------------ 18* ------------------------------------------------------
17* Arc90's Readability is designed to run in the browser. It works on the DOM 19* Arc90's Readability is designed to run in the browser. It works on the DOM
18* tree (the parsed HTML) after the page's CSS styles have been applied and 20* tree (the parsed HTML) after the page's CSS styles have been applied and
19* Javascript code executed. This PHP port does not run inside a browser. 21* Javascript code executed. This PHP port does not run inside a browser.
20* We use PHP's ability to parse HTML to build our DOM tree, but we cannot 22* We use PHP's ability to parse HTML to build our DOM tree, but we cannot
21* rely on CSS or Javascript support. As such, the results will not always 23* rely on CSS or Javascript support. As such, the results will not always
22* match Arc90's Readability. (For example, if a web page contains CSS style 24* match Arc90's Readability. (For example, if a web page contains CSS style
23* rules or Javascript code which hide certain HTML elements from display, 25* rules or Javascript code which hide certain HTML elements from display,
24* Arc90's Readability will dismiss those from consideration but our PHP port, 26* Arc90's Readability will dismiss those from consideration but our PHP port,
25* unable to understand CSS or Javascript, will not know any better.) 27* unable to understand CSS or Javascript, will not know any better.)
26* 28*
27* Another significant difference is that the aim of Arc90's Readability is 29* Another significant difference is that the aim of Arc90's Readability is
28* to re-present the main content block of a given web page so users can 30* to re-present the main content block of a given web page so users can
29* read it more easily in their browsers. Correct identification, clean up, 31* read it more easily in their browsers. Correct identification, clean up,
30* and separation of the content block is only a part of this process. 32* and separation of the content block is only a part of this process.
31* This PHP port is only concerned with this part, it does not include code 33* This PHP port is only concerned with this part, it does not include code
32* that relates to presentation in the browser - Arc90 already do 34* that relates to presentation in the browser - Arc90 already do
33* that extremely well, and for PDF output there's FiveFilters.org's 35* that extremely well, and for PDF output there's FiveFilters.org's
34* PDF Newspaper: http://fivefilters.org/pdf-newspaper/. 36* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
35* 37*
36* Finally, this class contains methods that might be useful for developers 38* Finally, this class contains methods that might be useful for developers
37* working on HTML document fragments. So without deviating too much from 39* working on HTML document fragments. So without deviating too much from
38* the original code (which I don't want to do because it makes debugging 40* the original code (which I don't want to do because it makes debugging
39* and updating more difficult), I've tried to make it a little more 41* and updating more difficult), I've tried to make it a little more
40* developer friendly. You should be able to use the methods here on 42* developer friendly. You should be able to use the methods here on
41* existing DOMElement objects without passing an entire HTML document to 43* existing DOMElement objects without passing an entire HTML document to
42* be parsed. 44* be parsed.
43*/ 45*/
44 46
@@ -46,13 +48,13 @@
46require_once(dirname(__FILE__).'/JSLikeHTMLElement.php'); 48require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
47 49
48// Alternative usage (for testing only!) 50// Alternative usage (for testing only!)
49// uncomment the lines below and call Readability.php in your browser 51// uncomment the lines below and call Readability.php in your browser
50// passing it the URL of the page you'd like content from, e.g.: 52// passing it the URL of the page you'd like content from, e.g.:
51// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php 53// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
52 54
53/* 55/*
54if (!isset($_GET['url']) || $_GET['url'] == '') { 56if (!isset($_GET['url']) || $_GET['url'] == '') {
55 die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html'); 57 die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
56} 58}
57$url = $_GET['url']; 59$url = $_GET['url'];
58if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url; 60if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
@@ -62,1043 +64,1074 @@ $r->init();
62echo $r->articleContent->innerHTML; 64echo $r->articleContent->innerHTML;
63*/ 65*/
64 66
65
66class Readability 67class Readability
67{ 68{
68 /* constants */ 69 public $version = '1.7.1-without-multi-page';
69 const FLAG_STRIP_UNLIKELYS = 1; 70 public $convertLinksToFootnotes = false;
70 const FLAG_WEIGHT_CLASSES = 2; 71 public $revertForcedParagraphElements = true;
71 const FLAG_CLEAN_CONDITIONALLY = 4; 72 public $articleTitle;
72 73 public $articleContent;
73 public $version = '1.7.1-without-multi-page'; 74 public $dom;
74 public $convertLinksToFootnotes = false; 75 public $url = null; // optional - URL where HTML was retrieved
75 public $revertForcedParagraphElements = true; 76 public $debug = false;
76 public $articleTitle; 77 public $lightClean = true; // preserves more content (experimental) added 2012-09-19
77 public $articleContent; 78 protected $body = null; //
78 public $dom; 79 protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
79 public $url = null; // optional - URL where HTML was retrieved 80 protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
80 public $debug = false; 81 protected $success = false; // indicates whether we were able to extract or not
81 protected $body = null; // 82
82 protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later 83 /**
83 protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. 84 * All of the regular expressions in use within readability.
84 protected $success = false; // indicates whether we were able to extract or not 85 * Defined up here so we don't instantiate them repeatedly in loops.
85 86 **/
86 /** 87 public $regexps = array(
87 * All of the regular expressions in use within readability. 88 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
88 * Defined up here so we don't instantiate them repeatedly in loops. 89 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
89 **/ 90 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
90 public $regexps = array( 91 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
91 'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i', 92 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
92 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', 93 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
93 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i', 94 'replaceFonts' => '/<(\/?)font[^>]*>/i',
94 'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', 95 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
95 'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i', 96 'normalize' => '/\s{2,}/',
96 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i', 97 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
97 'replaceFonts' => '/<(\/?)font[^>]*>/i', 98 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
98 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim() 99 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
99 'normalize' => '/\s{2,}/', 100 );
100 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/', 101
101 'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i', 102 /* constants */
102 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i' 103 const FLAG_STRIP_UNLIKELYS = 1;
103 ); 104 const FLAG_WEIGHT_CLASSES = 2;
104 105 const FLAG_CLEAN_CONDITIONALLY = 4;
105 /** 106
106 * Create instance of Readability 107 /**
107 * @param string UTF-8 encoded string 108 * Create instance of Readability
108 * @param string (optional) URL associated with HTML (used for footnotes) 109 * @param string UTF-8 encoded string
109 */ 110 * @param string (optional) URL associated with HTML (used for footnotes)
110 function __construct($html, $url=null) 111 * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
111 { 112 */
112 /* Turn all double br's into p's */ 113 function __construct($html, $url=null, $parser='libxml')
113 $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html); 114 {
114 $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); 115 $this->url = $url;
115 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); 116 /* Turn all double br's into p's */
116 $this->dom = new DOMDocument(); 117 $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
117 $this->dom->preserveWhiteSpace = false; 118 $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
118 $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); 119 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
119 if (trim($html) == '') $html = '<html></html>'; 120 if (trim($html) == '') $html = '<html></html>';
120 @$this->dom->loadHTML($html); 121 if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
121 $this->url = $url; 122 // all good
122 } 123 } else {
123 124 $this->dom = new DOMDocument();
124 /** 125 $this->dom->preserveWhiteSpace = false;
125 * Get article title element 126 @$this->dom->loadHTML($html);
126 * @return DOMElement 127 }
127 */ 128 $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
128 public function getTitle() { 129 }
129 return $this->articleTitle; 130
130 } 131 /**
131 132 * Get article title element
132 /** 133 * @return DOMElement
133 * Get article content element 134 */
134 * @return DOMElement 135 public function getTitle() {
135 */ 136 return $this->articleTitle;
136 public function getContent() { 137 }
137 return $this->articleContent; 138
138 } 139 /**
139 140 * Get article content element
140 /** 141 * @return DOMElement
141 * Runs readability. 142 */
142 * 143 public function getContent() {
143 * Workflow: 144 return $this->articleContent;
144 * 1. Prep the document by removing script tags, css, etc. 145 }
145 * 2. Build readability's DOM tree. 146
146 * 3. Grab the article content from the current dom tree. 147 /**
147 * 4. Replace the current DOM tree with the new one. 148 * Runs readability.
148 * 5. Read peacefully. 149 *
149 * 150 * Workflow:
150 * @return boolean true if we found content, false otherwise 151 * 1. Prep the document by removing script tags, css, etc.
151 **/ 152 * 2. Build readability's DOM tree.
152 public function init() 153 * 3. Grab the article content from the current dom tree.
153 { 154 * 4. Replace the current DOM tree with the new one.
154 if (!isset($this->dom->documentElement)) return false; 155 * 5. Read peacefully.
155 $this->removeScripts($this->dom); 156 *
156 //die($this->getInnerHTML($this->dom->documentElement)); 157 * @return boolean true if we found content, false otherwise
157 158 **/
158 // Assume successful outcome 159 public function init()
159 $this->success = true; 160 {
160 161 if (!isset($this->dom->documentElement)) return false;
161 $bodyElems = $this->dom->getElementsByTagName('body'); 162 $this->removeScripts($this->dom);
162 if ($bodyElems->length > 0) { 163 //die($this->getInnerHTML($this->dom->documentElement));
163 if ($this->bodyCache == null) { 164
164 $this->bodyCache = $bodyElems->item(0)->innerHTML; 165 // Assume successful outcome
165 } 166 $this->success = true;
166 if ($this->body == null) { 167
167 $this->body = $bodyElems->item(0); 168 $bodyElems = $this->dom->getElementsByTagName('body');
168 } 169 if ($bodyElems->length > 0) {
169 } 170 if ($this->bodyCache == null) {
170 171 $this->bodyCache = $bodyElems->item(0)->innerHTML;
171 $this->prepDocument(); 172 }
172 173 if ($this->body == null) {
173 //die($this->dom->documentElement->parentNode->nodeType); 174 $this->body = $bodyElems->item(0);
174 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); 175 }
175 //die($this->getInnerHTML($this->dom->documentElement)); 176 }
176 177
177 /* Build readability's DOM tree */ 178 $this->prepDocument();
178 $overlay = $this->dom->createElement('div'); 179
179 $innerDiv = $this->dom->createElement('div'); 180 //die($this->dom->documentElement->parentNode->nodeType);
180 $articleTitle = $this->getArticleTitle(); 181 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
181 $articleContent = $this->grabArticle(); 182 //die($this->getInnerHTML($this->dom->documentElement));
182 183
183 if (!$articleContent) { 184 /* Build readability's DOM tree */
184 $this->success = false; 185 $overlay = $this->dom->createElement('div');
185 $articleContent = $this->dom->createElement('div'); 186 $innerDiv = $this->dom->createElement('div');
186 $articleContent->setAttribute('id', 'readability-content'); 187 $articleTitle = $this->getArticleTitle();
187 $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>'; 188 $articleContent = $this->grabArticle();
188 return $this->success; 189
189 } 190 if (!$articleContent) {
190 191 $this->success = false;
191 $overlay->setAttribute('id', 'readOverlay'); 192 $articleContent = $this->dom->createElement('div');
192 $innerDiv->setAttribute('id', 'readInner'); 193 $articleContent->setAttribute('id', 'readability-content');
193 194 $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
194 /* Glue the structure of our document together. */ 195 }
195 $innerDiv->appendChild($articleTitle); 196
196 $innerDiv->appendChild($articleContent); 197 $overlay->setAttribute('id', 'readOverlay');
197 $overlay->appendChild($innerDiv); 198 $innerDiv->setAttribute('id', 'readInner');
198 199
199 /* Clear the old HTML, insert the new content. */ 200 /* Glue the structure of our document together. */
200 $this->body->innerHTML = ''; 201 $innerDiv->appendChild($articleTitle);
201 $this->body->appendChild($overlay); 202 $innerDiv->appendChild($articleContent);
202 //document.body.insertBefore(overlay, document.body.firstChild); 203 $overlay->appendChild($innerDiv);
203 $this->body->removeAttribute('style'); 204
204 205 /* Clear the old HTML, insert the new content. */
205 $this->postProcessContent($articleContent); 206 $this->body->innerHTML = '';
206 207 $this->body->appendChild($overlay);
207 // Set title and content instance variables 208 //document.body.insertBefore(overlay, document.body.firstChild);
208 $this->articleTitle = $articleTitle; 209 $this->body->removeAttribute('style');
209 $this->articleContent = $articleContent; 210
210 211 $this->postProcessContent($articleContent);
211 return $this->success; 212
212 } 213 // Set title and content instance variables
213 214 $this->articleTitle = $articleTitle;
214 /** 215 $this->articleContent = $articleContent;
215 * Debug 216
216 */ 217 return $this->success;
217 protected function dbg($msg) { 218 }
218 if ($this->debug) echo '* ',$msg, '<br />', "\n"; 219
219 } 220 /**
220 221 * Debug
221 /** 222 */
222 * Run any post-process modifications to article content as necessary. 223 protected function dbg($msg) {
223 * 224 if ($this->debug) echo '* ',$msg, "\n";
224 * @param DOMElement 225 }
225 * @return void 226
226 */ 227 /**
227 public function postProcessContent($articleContent) { 228 * Run any post-process modifications to article content as necessary.
228 if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { 229 *
229 $this->addFootnotes($articleContent); 230 * @param DOMElement
230 } 231 * @return void
231 } 232 */
232 233 public function postProcessContent($articleContent) {
233 /** 234 if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) {
234 * Get the article title as an H1. 235 $this->addFootnotes($articleContent);
235 * 236 }
236 * @return DOMElement 237 }
237 */ 238
238 protected function getArticleTitle() { 239 /**
239 $curTitle = ''; 240 * Get the article title as an H1.
240 $origTitle = ''; 241 *
241 242 * @return DOMElement
242 try { 243 */
243 $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0)); 244 protected function getArticleTitle() {
244 } catch(Exception $e) {} 245 $curTitle = '';
245 246 $origTitle = '';
246 if (preg_match('/ [\|\-] /', $curTitle)) 247
247 { 248 try {
248 $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle); 249 $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
249 250 } catch(Exception $e) {}
250 if (count(explode(' ', $curTitle)) < 3) { 251
251 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle); 252 if (preg_match('/ [\|\-] /', $curTitle))
252 } 253 {
253 } 254 $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
254 else if (strpos($curTitle, ': ') !== false) 255
255 { 256 if (count(explode(' ', $curTitle)) < 3) {
256 $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle); 257 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
257 258 }
258 if (count(explode(' ', $curTitle)) < 3) { 259 }
259 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle); 260 else if (strpos($curTitle, ': ') !== false)
260 } 261 {
261 } 262 $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
262 else if(strlen($curTitle) > 150 || strlen($curTitle) < 15) 263
263 { 264 if (count(explode(' ', $curTitle)) < 3) {
264 $hOnes = $this->dom->getElementsByTagName('h1'); 265 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
265 if($hOnes->length == 1) 266 }
266 { 267 }
267 $curTitle = $this->getInnerText($hOnes->item(0)); 268 else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)
268 } 269 {
269 } 270 $hOnes = $this->dom->getElementsByTagName('h1');
270 271 if($hOnes->length == 1)
271 $curTitle = trim($curTitle); 272 {
272 273 $curTitle = $this->getInnerText($hOnes->item(0));
273 if (count(explode(' ', $curTitle)) <= 4) { 274 }
274 $curTitle = $origTitle; 275 }
275 } 276
276 277 $curTitle = trim($curTitle);
277 $articleTitle = $this->dom->createElement('h1'); 278
278 $articleTitle->innerHTML = $curTitle; 279 if (count(explode(' ', $curTitle)) <= 4) {
279 280 $curTitle = $origTitle;
280 return $articleTitle; 281 }
281 } 282
282 283 $articleTitle = $this->dom->createElement('h1');
283 /** 284 $articleTitle->innerHTML = $curTitle;
284 * Prepare the HTML document for readability to scrape it. 285
285 * This includes things like stripping javascript, CSS, and handling terrible markup. 286 return $articleTitle;
286 * 287 }
287 * @return void 288
288 **/ 289 /**
289 protected function prepDocument() { 290 * Prepare the HTML document for readability to scrape it.
290 /** 291 * This includes things like stripping javascript, CSS, and handling terrible markup.
291 * In some cases a body element can't be found (if the HTML is totally hosed for example) 292 *
292 * so we create a new body node and append it to the document. 293 * @return void
293 */ 294 **/
294 if ($this->body == null) 295 protected function prepDocument() {
295 { 296 /**
296 $this->body = $this->dom->createElement('body'); 297 * In some cases a body element can't be found (if the HTML is totally hosed for example)
297 $this->dom->documentElement->appendChild($this->body); 298 * so we create a new body node and append it to the document.
298 } 299 */
299 $this->body->setAttribute('id', 'readabilityBody'); 300 if ($this->body == null)
300 301 {
301 /* Remove all style tags in head */ 302 $this->body = $this->dom->createElement('body');
302 $styleTags = $this->dom->getElementsByTagName('style'); 303 $this->dom->documentElement->appendChild($this->body);
303 for ($i = $styleTags->length-1; $i >= 0; $i--) 304 }
304 { 305 $this->body->setAttribute('id', 'readabilityBody');
305 $styleTags->item($i)->parentNode->removeChild($styleTags->item($i)); 306
306 } 307 /* Remove all style tags in head */
307 308 $styleTags = $this->dom->getElementsByTagName('style');
308 /* Turn all double br's into p's */ 309 for ($i = $styleTags->length-1; $i >= 0; $i--)
309 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */ 310 {
310 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>'); 311 $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
311 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. 312 }
312 // Manipulating innerHTML as it's done in JS is not possible in PHP. 313
313 } 314 /* Turn all double br's into p's */
314 315 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
315 /** 316 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
316 * For easier reading, convert this document to have footnotes at the bottom rather than inline links. 317 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
317 * @see http://www.roughtype.com/archives/2010/05/experiments_in.php 318 // Manipulating innerHTML as it's done in JS is not possible in PHP.
318 * 319 }
319 * @return void 320
320 **/ 321 /**
321 public function addFootnotes($articleContent) { 322 * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
322 $footnotesWrapper = $this->dom->createElement('div'); 323 * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
323 $footnotesWrapper->setAttribute('id', 'readability-footnotes'); 324 *
324 $footnotesWrapper->innerHTML = '<h3>References</h3>'; 325 * @return void
325 326 **/
326 $articleFootnotes = $this->dom->createElement('ol'); 327 public function addFootnotes($articleContent) {
327 $articleFootnotes->setAttribute('id', 'readability-footnotes-list'); 328 $footnotesWrapper = $this->dom->createElement('div');
328 $footnotesWrapper->appendChild($articleFootnotes); 329 $footnotesWrapper->setAttribute('id', 'readability-footnotes');
329 330 $footnotesWrapper->innerHTML = '<h3>References</h3>';
330 $articleLinks = $articleContent->getElementsByTagName('a'); 331
331 332 $articleFootnotes = $this->dom->createElement('ol');
332 $linkCount = 0; 333 $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
333 for ($i = 0; $i < $articleLinks->length; $i++) 334 $footnotesWrapper->appendChild($articleFootnotes);
334 { 335
335 $articleLink = $articleLinks->item($i); 336 $articleLinks = $articleContent->getElementsByTagName('a');
336 $footnoteLink = $articleLink->cloneNode(true); 337
337 $refLink = $this->dom->createElement('a'); 338 $linkCount = 0;
338 $footnote = $this->dom->createElement('li'); 339 for ($i = 0; $i < $articleLinks->length; $i++)
339 $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST); 340 {
340 if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST); 341 $articleLink = $articleLinks->item($i);
341 //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host, 342 $footnoteLink = $articleLink->cloneNode(true);
342 $linkText = $this->getInnerText($articleLink); 343 $refLink = $this->dom->createElement('a');
343 344 $footnote = $this->dom->createElement('li');
344 if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) { 345 $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
345 continue; 346 if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);
346 } 347 //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,
347 348 $linkText = $this->getInnerText($articleLink);
348 $linkCount++; 349
349 350 if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
350 /** Add a superscript reference after the article link */ 351 continue;
351 $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount); 352 }
352 $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>'; 353
353 $refLink->setAttribute('class', 'readability-DoNotFootnote'); 354 $linkCount++;
354 $refLink->setAttribute('style', 'color: inherit;'); 355
355 356 /** Add a superscript reference after the article link */
356 //TODO: does this work or should we use DOMNode.isSameNode()? 357 $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
357 if ($articleLink->parentNode->lastChild == $articleLink) { 358 $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
358 $articleLink->parentNode->appendChild($refLink); 359 $refLink->setAttribute('class', 'readability-DoNotFootnote');
359 } else { 360 $refLink->setAttribute('style', 'color: inherit;');
360 $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling); 361
361 } 362 //TODO: does this work or should we use DOMNode.isSameNode()?
362 363 if ($articleLink->parentNode->lastChild == $articleLink) {
363 $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;'); 364 $articleLink->parentNode->appendChild($refLink);
364 $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount); 365 } else {
365 366 $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
366 $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> '; 367 }
367 368
368 $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText); 369 $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
369 $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount); 370 $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
370 371
371 $footnote->appendChild($footnoteLink); 372 $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
372 if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>'; 373
373 374 $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);
374 $articleFootnotes->appendChild($footnote); 375 $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
375 } 376
376 377 $footnote->appendChild($footnoteLink);
377 if ($linkCount > 0) { 378 if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';
378 $articleContent->appendChild($footnotesWrapper); 379
379 } 380 $articleFootnotes->appendChild($footnote);
380 } 381 }
381 382
382 /** 383 if ($linkCount > 0) {
383 * Reverts P elements with class 'readability-styled' 384 $articleContent->appendChild($footnotesWrapper);
384 * to text nodes - which is what they were before. 385 }
385 * 386 }
386 * @param DOMElement 387
387 * @return void 388 /**
388 */ 389 * Reverts P elements with class 'readability-styled'
389 function revertReadabilityStyledElements($articleContent) { 390 * to text nodes - which is what they were before.
390 $xpath = new DOMXPath($articleContent->ownerDocument); 391 *
391 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent); 392 * @param DOMElement
392 //$elems = $articleContent->getElementsByTagName('p'); 393 * @return void
393 for ($i = $elems->length-1; $i >= 0; $i--) { 394 */
394 $e = $elems->item($i); 395 function revertReadabilityStyledElements($articleContent) {
395 $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); 396 $xpath = new DOMXPath($articleContent->ownerDocument);
396 //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') { 397 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
397 // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e); 398 //$elems = $articleContent->getElementsByTagName('p');
398 //} 399 for ($i = $elems->length-1; $i >= 0; $i--) {
399 } 400 $e = $elems->item($i);
400 } 401 $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
401 402 //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
402 /** 403 // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
403 * Prepare the article node for display. Clean out any inline styles, 404 //}
404 * iframes, forms, strip extraneous <p> tags, etc. 405 }
405 * 406 }
406 * @param DOMElement 407
407 * @return void 408 /**
408 */ 409 * Prepare the article node for display. Clean out any inline styles,
409 function prepArticle($articleContent) { 410 * iframes, forms, strip extraneous <p> tags, etc.
410 $this->cleanStyles($articleContent); 411 *
411 $this->killBreaks($articleContent); 412 * @param DOMElement
412 if ($this->revertForcedParagraphElements) { 413 * @return void
413 $this->revertReadabilityStyledElements($articleContent); 414 */
414 } 415 function prepArticle($articleContent) {
415 416 $this->cleanStyles($articleContent);
416 /* Clean out junk from the article content */ 417 $this->killBreaks($articleContent);
417 $this->cleanConditionally($articleContent, 'form'); 418 if ($this->revertForcedParagraphElements) {
418 $this->clean($articleContent, 'object'); 419 $this->revertReadabilityStyledElements($articleContent);
419 $this->clean($articleContent, 'h1'); 420 }
420 421
421 /** 422 /* Clean out junk from the article content */
422 * If there is only one h2, they are probably using it 423 $this->cleanConditionally($articleContent, 'form');
423 * as a header and not a subheader, so remove it since we already have a header. 424 $this->clean($articleContent, 'object');
424 ***/ 425 $this->clean($articleContent, 'h1');
425 if ($articleContent->getElementsByTagName('h2')->length == 1) { 426
426 $this->clean($articleContent, 'h2'); 427 /**
427 } 428 * If there is only one h2, they are probably using it
428 $this->clean($articleContent, 'iframe'); 429 * as a header and not a subheader, so remove it since we already have a header.
429 430 ***/
430 $this->cleanHeaders($articleContent); 431 if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
431 432 $this->clean($articleContent, 'h2');
432 /* Do these last as the previous stuff may have removed junk that will affect these */ 433 }
433 $this->cleanConditionally($articleContent, 'table'); 434 $this->clean($articleContent, 'iframe');
434 $this->cleanConditionally($articleContent, 'ul'); 435
435 $this->cleanConditionally($articleContent, 'div'); 436 $this->cleanHeaders($articleContent);
436 437
437 /* Remove extra paragraphs */ 438 /* Do these last as the previous stuff may have removed junk that will affect these */
438 $articleParagraphs = $articleContent->getElementsByTagName('p'); 439 $this->cleanConditionally($articleContent, 'table');
439 for ($i = $articleParagraphs->length-1; $i >= 0; $i--) 440 $this->cleanConditionally($articleContent, 'ul');
440 { 441 $this->cleanConditionally($articleContent, 'div');
441 $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length; 442
442 $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length; 443 /* Remove extra paragraphs */
443 $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length; 444 $articleParagraphs = $articleContent->getElementsByTagName('p');
444 445 for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
445 if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '') 446 {
446 { 447 $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
447 $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i)); 448 $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
448 } 449 $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
449 } 450 $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
450 451
451 try { 452 if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
452 $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML); 453 {
453 //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p'); 454 $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
454 } 455 }
455 catch (Exception $e) { 456 }
456 $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e); 457
457 } 458 try {
458 } 459 $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);
459 460 //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
460 /** 461 }
461 * Initialize a node with the readability object. Also checks the 462 catch (Exception $e) {
462 * className/id for special names to add to its score. 463 $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
463 * 464 }
464 * @param Element 465 }
465 * @return void 466
466 **/ 467 /**
467 protected function initializeNode($node) { 468 * Initialize a node with the readability object. Also checks the
468 $readability = $this->dom->createAttribute('readability'); 469 * className/id for special names to add to its score.
469 $readability->value = 0; // this is our contentScore 470 *
470 $node->setAttributeNode($readability); 471 * @param Element
471 472 * @return void
472 switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case 473 **/
473 case 'DIV': 474 protected function initializeNode($node) {
474 $readability->value += 5; 475 $readability = $this->dom->createAttribute('readability');
475 break; 476 $readability->value = 0; // this is our contentScore
476 477 $node->setAttributeNode($readability);
477 case 'PRE': 478
478 case 'TD': 479 switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
479 case 'BLOCKQUOTE': 480 case 'DIV':
480 $readability->value += 3; 481 $readability->value += 5;
481 break; 482 break;
482 483
483 case 'ADDRESS': 484 case 'PRE':
484 case 'OL': 485 case 'TD':
485 case 'UL': 486 case 'BLOCKQUOTE':
486 case 'DL': 487 $readability->value += 3;
487 case 'DD': 488 break;
488 case 'DT': 489
489 case 'LI': 490 case 'ADDRESS':
490 case 'FORM': 491 case 'OL':
491 $readability->value -= 3; 492 case 'UL':
492 break; 493 case 'DL':
493 494 case 'DD':
494 case 'H1': 495 case 'DT':
495 case 'H2': 496 case 'LI':
496 case 'H3': 497 case 'FORM':
497 case 'H4': 498 $readability->value -= 3;
498 case 'H5': 499 break;
499 case 'H6': 500
500 case 'TH': 501 case 'H1':
501 $readability->value -= 5; 502 case 'H2':
502 break; 503 case 'H3':
503 } 504 case 'H4':
504 $readability->value += $this->getClassWeight($node); 505 case 'H5':
505 } 506 case 'H6':
506 507 case 'TH':
507 /*** 508 $readability->value -= 5;
508 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is 509 break;
509 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. 510 }
510 * 511 $readability->value += $this->getClassWeight($node);
511 * @return DOMElement 512 }
512 **/ 513
513 protected function grabArticle($page=null) { 514 /***
514 $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS); 515 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
515 if (!$page) $page = $this->dom; 516 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
516 $allElements = $page->getElementsByTagName('*'); 517 *
517 /** 518 * @return DOMElement
518 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs 519 **/
519 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.) 520 protected function grabArticle($page=null) {
520 * 521 $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
521 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 522 if (!$page) $page = $this->dom;
522 * TODO: Shouldn't this be a reverse traversal? 523 $allElements = $page->getElementsByTagName('*');
523 **/ 524 /**
524 $node = null; 525 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
525 $nodesToScore = array(); 526 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
526 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) { 527 *
527 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) { 528 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
528 //$node = $targetList->item($nodeIndex); 529 * TODO: Shouldn't this be a reverse traversal?
529 $tagName = strtoupper($node->tagName); 530 **/
530 /* Remove unlikely candidates */ 531 $node = null;
531 if ($stripUnlikelyCandidates) { 532 $nodesToScore = array();
532 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id'); 533 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
533 if ( 534 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
534 preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) && 535 //$node = $targetList->item($nodeIndex);
535 !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) && 536 $tagName = strtoupper($node->tagName);
536 $tagName != 'BODY' 537 /* Remove unlikely candidates */
537 ) 538 if ($stripUnlikelyCandidates) {
538 { 539 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
539 $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString); 540 if (
540 //$nodesToRemove[] = $node; 541 preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
541 $node->parentNode->removeChild($node); 542 !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
542 $nodeIndex--; 543 $tagName != 'BODY'
543 continue; 544 )
544 } 545 {
545 } 546 $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
546 547 //$nodesToRemove[] = $node;
547 if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') { 548 $node->parentNode->removeChild($node);
548 $nodesToScore[] = $node; 549 $nodeIndex--;
549 } 550 continue;
550 551 }
551 /* Turn all divs that don't have children block level elements into p's */ 552 }
552 if ($tagName == 'DIV') { 553
553 if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) { 554 if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
554 //$this->dbg('Altering div to p'); 555 $nodesToScore[] = $node;
555 $newNode = $this->dom->createElement('p'); 556 }
556 try { 557
557 $newNode->innerHTML = $node->innerHTML; 558 /* Turn all divs that don't have children block level elements into p's */
558 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node); 559 if ($tagName == 'DIV') {
559 $node->parentNode->replaceChild($newNode, $node); 560 if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
560 $nodeIndex--; 561 //$this->dbg('Altering div to p');
561 $nodesToScore[] = $node; // or $newNode? 562 $newNode = $this->dom->createElement('p');
562 } 563 try {
563 catch(Exception $e) { 564 $newNode->innerHTML = $node->innerHTML;
564 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e); 565 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
565 } 566 $node->parentNode->replaceChild($newNode, $node);
566 } 567 $nodeIndex--;
567 else 568 $nodesToScore[] = $node; // or $newNode?
568 { 569 }
569 // EXPERIMENTAL 570 catch(Exception $e) {
570 // TODO: change these p elements back to text nodes after processing 571 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
571 for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) { 572 }
572 $childNode = $node->childNodes->item($i); 573 }
573 if ($childNode->nodeType == 3) { // XML_TEXT_NODE 574 else
574 //$this->dbg('replacing text node with a p tag with the same content.'); 575 {
575 $p = $this->dom->createElement('p'); 576 /* EXPERIMENTAL */
576 $p->innerHTML = $childNode->nodeValue; 577 // TODO: change these p elements back to text nodes after processing
577 $p->setAttribute('style', 'display: inline;'); 578 for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
578 $p->setAttribute('class', 'readability-styled'); 579 $childNode = $node->childNodes->item($i);
579 $childNode->parentNode->replaceChild($p, $childNode); 580 if ($childNode->nodeType == 3) { // XML_TEXT_NODE
580 } 581 //$this->dbg('replacing text node with a p tag with the same content.');
581 } 582 $p = $this->dom->createElement('p');
582 } 583 $p->innerHTML = $childNode->nodeValue;
583 } 584 $p->setAttribute('style', 'display: inline;');
584 } 585 $p->setAttribute('class', 'readability-styled');
585 586 $childNode->parentNode->replaceChild($p, $childNode);
586 /** 587 }
587 * Loop through all paragraphs, and assign a score to them based on how content-y they look. 588 }
588 * Then add their score to their parent node. 589 }
589 * 590 }
590 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. 591 }
591 **/ 592
592 $candidates = array(); 593 /**
593 for ($pt=0; $pt < count($nodesToScore); $pt++) { 594 * Loop through all paragraphs, and assign a score to them based on how content-y they look.
594 $parentNode = $nodesToScore[$pt]->parentNode; 595 * Then add their score to their parent node.
595 // $grandParentNode = $parentNode ? $parentNode->parentNode : null; 596 *
596 $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null); 597 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
597 $innerText = $this->getInnerText($nodesToScore[$pt]); 598 **/
598 599 $candidates = array();
599 if (!$parentNode || !isset($parentNode->tagName)) { 600 for ($pt=0; $pt < count($nodesToScore); $pt++) {
600 continue; 601 $parentNode = $nodesToScore[$pt]->parentNode;
601 } 602 // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
602 603 $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
603 /* If this paragraph is less than 25 characters, don't even count it. */ 604 $innerText = $this->getInnerText($nodesToScore[$pt]);
604 if(strlen($innerText) < 25) { 605
605 continue; 606 if (!$parentNode || !isset($parentNode->tagName)) {
606 } 607 continue;
607 608 }
608 /* Initialize readability data for the parent. */ 609
609 if (!$parentNode->hasAttribute('readability')) 610 /* If this paragraph is less than 25 characters, don't even count it. */
610 { 611 if(strlen($innerText) < 25) {
611 $this->initializeNode($parentNode); 612 continue;
612 $candidates[] = $parentNode; 613 }
613 } 614
614 615 /* Initialize readability data for the parent. */
615 /* Initialize readability data for the grandparent. */ 616 if (!$parentNode->hasAttribute('readability'))
616 if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) 617 {
617 { 618 $this->initializeNode($parentNode);
618 $this->initializeNode($grandParentNode); 619 $candidates[] = $parentNode;
619 $candidates[] = $grandParentNode; 620 }
620 } 621
621 622 /* Initialize readability data for the grandparent. */
622 $contentScore = 0; 623 if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
623 624 {
624 /* Add a point for the paragraph itself as a base. */ 625 $this->initializeNode($grandParentNode);
625 $contentScore++; 626 $candidates[] = $grandParentNode;
626 627 }
627 /* Add points for any commas within this paragraph */ 628
628 $contentScore += count(explode(',', $innerText)); 629 $contentScore = 0;
629 630
630 /* For every 100 characters in this paragraph, add another point. Up to 3 points. */ 631 /* Add a point for the paragraph itself as a base. */
631 $contentScore += min(floor(strlen($innerText) / 100), 3); 632 $contentScore++;
632 633
633 /* Add the score to the parent. The grandparent gets half. */ 634 /* Add points for any commas within this paragraph */
634 $parentNode->getAttributeNode('readability')->value += $contentScore; 635 $contentScore += count(explode(',', $innerText));
635 636
636 if ($grandParentNode) { 637 /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
637 $grandParentNode->getAttributeNode('readability')->value += $contentScore/2; 638 $contentScore += min(floor(strlen($innerText) / 100), 3);
638 } 639
639 } 640 /* Add the score to the parent. The grandparent gets half. */
640 641 $parentNode->getAttributeNode('readability')->value += $contentScore;
641 /** 642
642 * After we've calculated scores, loop through all of the possible candidate nodes we found 643 if ($grandParentNode) {
643 * and find the one with the highest score. 644 $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
644 **/ 645 }
645 $topCandidate = null; 646 }
646 for ($c=0, $cl=count($candidates); $c < $cl; $c++) 647
647 { 648 /**
648 /** 649 * After we've calculated scores, loop through all of the possible candidate nodes we found
649 * Scale the final candidates score based on link density. Good content should have a 650 * and find the one with the highest score.
650 * relatively small link density (5% or less) and be mostly unaffected by this operation. 651 **/
651 **/ 652 $topCandidate = null;
652 $readability = $candidates[$c]->getAttributeNode('readability'); 653 for ($c=0, $cl=count($candidates); $c < $cl; $c++)
653 $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c])); 654 {
654 655 /**
655 $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value); 656 * Scale the final candidates score based on link density. Good content should have a
656 657 * relatively small link density (5% or less) and be mostly unaffected by this operation.
657 if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) { 658 **/
658 $topCandidate = $candidates[$c]; 659 $readability = $candidates[$c]->getAttributeNode('readability');
659 } 660 $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
660 } 661
661 662 $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
662 /** 663
663 * If we still have no top candidate, just use the body as a last resort. 664 if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
664 * We also have to copy the body node so it is something we can modify. 665 $topCandidate = $candidates[$c];
665 **/ 666 }
666 if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY') 667 }
667 { 668
668 $topCandidate = $this->dom->createElement('div'); 669 /**
669 if ($page instanceof DOMDocument) { 670 * If we still have no top candidate, just use the body as a last resort.
670 if (!isset($page->documentElement)) { 671 * We also have to copy the body node so it is something we can modify.
671 // we don't have a body either? what a mess! :) 672 **/
672 } else { 673 if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
673 $topCandidate->innerHTML = $page->documentElement->innerHTML; 674 {
674 $page->documentElement->innerHTML = ''; 675 $topCandidate = $this->dom->createElement('div');
675 $page->documentElement->appendChild($topCandidate); 676 if ($page instanceof DOMDocument) {
676 } 677 if (!isset($page->documentElement)) {
677 } else { 678 // we don't have a body either? what a mess! :)
678 $topCandidate->innerHTML = $page->innerHTML; 679 } else {
679 $page->innerHTML = ''; 680 $topCandidate->innerHTML = $page->documentElement->innerHTML;
680 $page->appendChild($topCandidate); 681 $page->documentElement->innerHTML = '';
681 } 682 $page->documentElement->appendChild($topCandidate);
682 $this->initializeNode($topCandidate); 683 }
683 } 684 } else {
684 685 $topCandidate->innerHTML = $page->innerHTML;
685 /** 686 $page->innerHTML = '';
686 * Now that we have the top candidate, look through its siblings for content that might also be related. 687 $page->appendChild($topCandidate);
687 * Things like preambles, content split by ads that we removed, etc. 688 }
688 **/ 689 $this->initializeNode($topCandidate);
689 $articleContent = $this->dom->createElement('div'); 690 }
690 $articleContent->setAttribute('id', 'readability-content'); 691
691 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2); 692 /**
692 $siblingNodes = $topCandidate->parentNode->childNodes; 693 * Now that we have the top candidate, look through its siblings for content that might also be related.
693 if (!isset($siblingNodes)) { 694 * Things like preambles, content split by ads that we removed, etc.
694 $siblingNodes = new stdClass; 695 **/
695 $siblingNodes->length = 0; 696 $articleContent = $this->dom->createElement('div');
696 } 697 $articleContent->setAttribute('id', 'readability-content');
697 698 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
698 for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++) 699 $siblingNodes = $topCandidate->parentNode->childNodes;
699 { 700 if (!isset($siblingNodes)) {
700 $siblingNode = $siblingNodes->item($s); 701 $siblingNodes = new stdClass;
701 $append = false; 702 $siblingNodes->length = 0;
702 703 }
703 $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : '')); 704
704 705 for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
705 //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown')); 706 {
706 707 $siblingNode = $siblingNodes->item($s);
707 if ($siblingNode === $topCandidate) 708 $append = false;
708 // or if ($siblingNode->isSameNode($topCandidate)) 709
709 { 710 $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
710 $append = true; 711
711 } 712 //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
712 713
713 $contentBonus = 0; 714 if ($siblingNode === $topCandidate)
714 /* Give a bonus if sibling nodes and top candidates have the example same classname */ 715 // or if ($siblingNode->isSameNode($topCandidate))
715 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') { 716 {
716 $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2; 717 $append = true;
717 } 718 }
718 719
719 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) 720 $contentBonus = 0;
720 { 721 /* Give a bonus if sibling nodes and top candidates have the example same classname */
721 $append = true; 722 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
722 } 723 $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
723 724 }
724 if (strtoupper($siblingNode->nodeName) == 'P') { 725
725 $linkDensity = $this->getLinkDensity($siblingNode); 726 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
726 $nodeContent = $this->getInnerText($siblingNode); 727 {
727 $nodeLength = strlen($nodeContent); 728 $append = true;
728 729 }
729 if ($nodeLength > 80 && $linkDensity < 0.25) 730
730 { 731 if (strtoupper($siblingNode->nodeName) == 'P') {
731 $append = true; 732 $linkDensity = $this->getLinkDensity($siblingNode);
732 } 733 $nodeContent = $this->getInnerText($siblingNode);
733 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) 734 $nodeLength = strlen($nodeContent);
734 { 735
735 $append = true; 736 if ($nodeLength > 80 && $linkDensity < 0.25)
736 } 737 {
737 } 738 $append = true;
738 739 }
739 /* Look for a special classname */ 740 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
740 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('class') && $siblingNode->getAttribute('class') != '') 741 {
741 { 742 $append = true;
742 if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('class'))) { 743 }
743 $append = true; 744 }
744 } 745
745 } 746 if ($append)
746 747 {
747 /* Look for a special classname */ 748 $this->dbg('Appending node: ' . $siblingNode->nodeName);
748 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('id') && $siblingNode->getAttribute('id') != '') 749
749 { 750 $nodeToAppend = null;
750 if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('id'))) { 751 $sibNodeName = strtoupper($siblingNode->nodeName);
751 $append = true; 752 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
752 } 753 /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
753 } 754
754 755 $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
755 756 $nodeToAppend = $this->dom->createElement('div');
756 if ($append) 757 try {
757 { 758 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
758 $this->dbg('Appending node: ' . $siblingNode->nodeName); 759 $nodeToAppend->innerHTML = $siblingNode->innerHTML;
759 760 }
760 $nodeToAppend = null; 761 catch(Exception $e)
761 $sibNodeName = strtoupper($siblingNode->nodeName); 762 {
762 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') { 763 $this->dbg('Could not alter siblingNode to div, reverting back to original.');
763 /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */ 764 $nodeToAppend = $siblingNode;
764 765 $s--;
765 $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.'); 766 $sl--;
766 $nodeToAppend = $this->dom->createElement('div'); 767 }
767 try { 768 } else {
768 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id')); 769 $nodeToAppend = $siblingNode;
769 $nodeToAppend->innerHTML = $siblingNode->innerHTML; 770 $s--;
770 } 771 $sl--;
771 catch(Exception $e) 772 }
772 { 773
773 $this->dbg('Could not alter siblingNode to div, reverting back to original.'); 774 /* To ensure a node does not interfere with readability styles, remove its classnames */
774 $nodeToAppend = $siblingNode; 775 $nodeToAppend->removeAttribute('class');
775 $s--; 776
776 $sl--; 777 /* Append sibling and subtract from our list because it removes the node when you append to another node */
777 } 778 $articleContent->appendChild($nodeToAppend);
778 } else { 779 }
779 $nodeToAppend = $siblingNode; 780 }
780 $s--; 781
781 $sl--; 782 /**
782 } 783 * So we have all of the content that we need. Now we clean it up for presentation.
783 784 **/
784 /* To ensure a node does not interfere with readability styles, remove its classnames */ 785 $this->prepArticle($articleContent);
785 $nodeToAppend->removeAttribute('class'); 786
786 787 /**
787 /* Append sibling and subtract from our list because it removes the node when you append to another node */ 788 * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
788 $articleContent->appendChild($nodeToAppend); 789 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
789 } 790 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
790 } 791 * finding the -right- content.
791 792 **/
792 /** 793 if (strlen($this->getInnerText($articleContent, false)) < 250)
793 * So we have all of the content that we need. Now we clean it up for presentation. 794 {
794 **/ 795 // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
795 $this->prepArticle($articleContent); 796 // in the meantime, we check and create an empty element if it's not there.
796 797 if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');
797 /** 798 $this->body->innerHTML = $this->bodyCache;
798 * Now that we've gone through the full algorithm, check to see if we got any meaningful content. 799
799 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher 800 if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
800 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of 801 $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
801 * finding the -right- content. 802 return $this->grabArticle($this->body);
802 **/ 803 }
803 if (strlen($this->getInnerText($articleContent, false)) < 250) 804 else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
804 { 805 $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
805 // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7 806 return $this->grabArticle($this->body);
806 // in the meantime, we check and create an empty element if it's not there. 807 }
807 if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body'); 808 else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
808 $this->body->innerHTML = $this->bodyCache; 809 $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
809 810 return $this->grabArticle($this->body);
810 if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) { 811 }
811 $this->removeFlag(self::FLAG_STRIP_UNLIKELYS); 812 else {
812 return $this->grabArticle($this->body); 813 return false;
813 } 814 }
814 else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { 815 }
815 $this->removeFlag(self::FLAG_WEIGHT_CLASSES); 816 return $articleContent;
816 return $this->grabArticle($this->body); 817 }
817 } 818
818 else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { 819 /**
819 $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY); 820 * Remove script tags from document
820 return $this->grabArticle($this->body); 821 *
821 } 822 * @param DOMElement
822 else { 823 * @return void
823 return false; 824 */
824 } 825 public function removeScripts($doc) {
825 } 826 $scripts = $doc->getElementsByTagName('script');
826 return $articleContent; 827 for($i = $scripts->length-1; $i >= 0; $i--)
827 } 828 {
828 829 $scripts->item($i)->parentNode->removeChild($scripts->item($i));
829 /** 830 }
830 * Remove script tags from document 831 }
831 * 832
832 * @param DOMElement 833 /**
833 * @return void 834 * Get the inner text of a node.
834 */ 835 * This also strips out any excess whitespace to be found.
835 public function removeScripts($doc) { 836 *
836 $scripts = $doc->getElementsByTagName('script'); 837 * @param DOMElement $
837 for($i = $scripts->length-1; $i >= 0; $i--) 838 * @param boolean $normalizeSpaces (default: true)
838 { 839 * @return string
839 $scripts->item($i)->parentNode->removeChild($scripts->item($i)); 840 **/
840 } 841 public function getInnerText($e, $normalizeSpaces=true) {
841 } 842 $textContent = '';
842 843
843 /** 844 if (!isset($e->textContent) || $e->textContent == '') {
844 * Get the inner text of a node. 845 return '';
845 * This also strips out any excess whitespace to be found. 846 }
846 * 847
847 * @param DOMElement $ 848 $textContent = trim($e->textContent);
848 * @param boolean $normalizeSpaces (default: true) 849
849 * @return string 850 if ($normalizeSpaces) {
850 **/ 851 return preg_replace($this->regexps['normalize'], ' ', $textContent);
851 public function getInnerText($e, $normalizeSpaces=true) { 852 } else {
852 $textContent = ''; 853 return $textContent;
853 854 }
854 if (!isset($e->textContent) || $e->textContent == '') { 855 }
855 return ''; 856
856 } 857 /**
857 858 * Get the number of times a string $s appears in the node $e.
858 $textContent = trim($e->textContent); 859 *
859 860 * @param DOMElement $e
860 if ($normalizeSpaces) { 861 * @param string - what to count. Default is ","
861 return preg_replace($this->regexps['normalize'], ' ', $textContent); 862 * @return number (integer)
862 } else { 863 **/
863 return $textContent; 864 public function getCharCount($e, $s=',') {
864 } 865 return substr_count($this->getInnerText($e), $s);
865 } 866 }
866 867
867 /** 868 /**
868 * Get the number of times a string $s appears in the node $e. 869 * Remove the style attribute on every $e and under.
869 * 870 *
870 * @param DOMElement $e 871 * @param DOMElement $e
871 * @param string - what to count. Default is "," 872 * @return void
872 * @return number (integer) 873 */
873 **/ 874 public function cleanStyles($e) {
874 public function getCharCount($e, $s=',') { 875 if (!is_object($e)) return;
875 return substr_count($this->getInnerText($e), $s); 876 $elems = $e->getElementsByTagName('*');
876 } 877 foreach ($elems as $elem) {
877 878 $elem->removeAttribute('style');
878 /** 879 }
879 * Remove the style attribute on every $e and under. 880 }
880 * 881
881 * @param DOMElement $e 882 /**
882 * @return void 883 * Get the density of links as a percentage of the content
883 */ 884 * This is the amount of text that is inside a link divided by the total text in the node.
884 public function cleanStyles($e) { 885 *
885 if (!is_object($e)) return; 886 * @param DOMElement $e
886 $elems = $e->getElementsByTagName('*'); 887 * @return number (float)
887 foreach ($elems as $elem) { 888 */
888 $elem->removeAttribute('style'); 889 public function getLinkDensity($e) {
889 } 890 $links = $e->getElementsByTagName('a');
890 } 891 $textLength = strlen($this->getInnerText($e));
891 892 $linkLength = 0;
892 /** 893 for ($i=0, $il=$links->length; $i < $il; $i++)
893 * Get the density of links as a percentage of the content 894 {
894 * This is the amount of text that is inside a link divided by the total text in the node. 895 $linkLength += strlen($this->getInnerText($links->item($i)));
895 * 896 }
896 * @param DOMElement $e 897 if ($textLength > 0) {
897 * @return number (float) 898 return $linkLength / $textLength;
898 */ 899 } else {
899 public function getLinkDensity($e) { 900 return 0;
900 $links = $e->getElementsByTagName('a'); 901 }
901 $textLength = strlen($this->getInnerText($e)); 902 }
902 $linkLength = 0; 903
903 for ($i=0, $il=$links->length; $i < $il; $i++) 904 /**
904 { 905 * Get an elements class/id weight. Uses regular expressions to tell if this
905 $linkLength += strlen($this->getInnerText($links->item($i))); 906 * element looks good or bad.
906 } 907 *
907 if ($textLength > 0) { 908 * @param DOMElement $e
908 return $linkLength / $textLength; 909 * @return number (Integer)
909 } else { 910 */
910 return 0; 911 public function getClassWeight($e) {
911 } 912 if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
912 } 913 return 0;
913 914 }
914 /** 915
915 * Get an elements class/id weight. Uses regular expressions to tell if this 916 $weight = 0;
916 * element looks good or bad. 917
917 * 918 /* Look for a special classname */
918 * @param DOMElement $e 919 if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
919 * @return number (Integer) 920 {
920 */ 921 if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
921 public function getClassWeight($e) { 922 $weight -= 25;
922 if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { 923 }
923 return 0; 924 if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
924 } 925 $weight += 25;
925 926 }
926 $weight = 0; 927 }
927 928
928 /* Look for a special classname */ 929 /* Look for a special ID */
929 if ($e->hasAttribute('class') && $e->getAttribute('class') != '') 930 if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
930 { 931 {
931 if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) { 932 if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
932 $weight -= 25; 933 $weight -= 25;
933 } 934 }
934 if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) { 935 if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
935 $weight += 25; 936 $weight += 25;
936 } 937 }
937 } 938 }
938 939 return $weight;
939 /* Look for a special ID */ 940 }
940 if ($e->hasAttribute('id') && $e->getAttribute('id') != '') 941
941 { 942 /**
942 if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) { 943 * Remove extraneous break tags from a node.
943 $weight -= 25; 944 *
944 } 945 * @param DOMElement $node
945 if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) { 946 * @return void
946 $weight += 25; 947 */
947 } 948 public function killBreaks($node) {
948 } 949 $html = $node->innerHTML;
949 return $weight; 950 $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
950 } 951 $node->innerHTML = $html;
951 952 }
952 /** 953
953 * Remove extraneous break tags from a node. 954 /**
954 * 955 * Clean a node of all elements of type "tag".
955 * @param DOMElement $node 956 * (Unless it's a youtube/vimeo video. People love movies.)
956 * @return void 957 *
957 */ 958 * Updated 2012-09-18 to preserve youtube/vimeo iframes
958 public function killBreaks($node) { 959 *
959 $html = $node->innerHTML; 960 * @param DOMElement $e
960 $html = preg_replace($this->regexps['killBreaks'], '<br />', $html); 961 * @param string $tag
961 $node->innerHTML = $html; 962 * @return void
962 } 963 */
963 964 public function clean($e, $tag) {
964 /** 965 $targetList = $e->getElementsByTagName($tag);
965 * Clean a node of all elements of type "tag". 966 $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
966 * (Unless it's a youtube/vimeo video. People love movies.) 967
967 * 968 for ($y=$targetList->length-1; $y >= 0; $y--) {
968 * @param DOMElement $e 969 /* Allow youtube and vimeo videos through as people usually want to see those. */
969 * @param string $tag 970 if ($isEmbed) {
970 * @return void 971 $attributeValues = '';
971 */ 972 for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
972 public function clean($e, $tag) { 973 $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
973 $targetList = $e->getElementsByTagName($tag); 974 }
974 $isEmbed = ($tag == 'object' || $tag == 'embed'); 975
975 976 /* First, check the elements attributes to see if any of them contain youtube or vimeo */
976 for ($y=$targetList->length-1; $y >= 0; $y--) { 977 if (preg_match($this->regexps['video'], $attributeValues)) {
977 /* Allow youtube and vimeo videos through as people usually want to see those. */ 978 continue;
978 if ($isEmbed) { 979 }
979 $attributeValues = ''; 980
980 for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) { 981 /* Then check the elements inside this element for the same. */
981 $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test) 982 if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
982 } 983 continue;
983 984 }
984 /* First, check the elements attributes to see if any of them contain youtube or vimeo */ 985 }
985 if (preg_match($this->regexps['video'], $attributeValues)) { 986 $targetList->item($y)->parentNode->removeChild($targetList->item($y));
986 continue; 987 }
987 } 988 }
988 989
989 /* Then check the elements inside this element for the same. */ 990 /**
990 if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) { 991 * Clean an element of all tags of type "tag" if they look fishy.
991 continue; 992 * "Fishy" is an algorithm based on content length, classnames,
992 } 993 * link density, number of images & embeds, etc.
993 } 994 *
994 $targetList->item($y)->parentNode->removeChild($targetList->item($y)); 995 * @param DOMElement $e
995 } 996 * @param string $tag
996 } 997 * @return void
997 998 */
998 /** 999 public function cleanConditionally($e, $tag) {
999 * Clean an element of all tags of type "tag" if they look fishy. 1000 if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
1000 * "Fishy" is an algorithm based on content length, classnames, 1001 return;
1001 * link density, number of images & embeds, etc. 1002 }
1002 * 1003
1003 * @param DOMElement $e 1004 $tagsList = $e->getElementsByTagName($tag);
1004 * @param string $tag 1005 $curTagsLength = $tagsList->length;
1005 * @return void 1006
1006 */ 1007 /**
1007 public function cleanConditionally($e, $tag) { 1008 * Gather counts for other typical elements embedded within.
1008 if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { 1009 * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
1009 return; 1010 *
1010 } 1011 * TODO: Consider taking into account original contentScore here.
1011 1012 */
1012 $tagsList = $e->getElementsByTagName($tag); 1013 for ($i=$curTagsLength-1; $i >= 0; $i--) {
1013 $curTagsLength = $tagsList->length; 1014 $weight = $this->getClassWeight($tagsList->item($i));
1014 1015 $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
1015 /** 1016
1016 * Gather counts for other typical elements embedded within. 1017 $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
1017 * Traverse backwards so we can remove nodes at the same time without effecting the traversal. 1018
1018 * 1019 if ($weight + $contentScore < 0) {
1019 * TODO: Consider taking into account original contentScore here. 1020 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1020 */ 1021 }
1021 for ($i=$curTagsLength-1; $i >= 0; $i--) { 1022 else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
1022 $weight = $this->getClassWeight($tagsList->item($i)); 1023 /**
1023 $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0; 1024 * If there are not very many commas, and the number of
1024 1025 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
1025 $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : '')); 1026 **/
1026 1027 $p = $tagsList->item($i)->getElementsByTagName('p')->length;
1027 if ($weight + $contentScore < 0) { 1028 $img = $tagsList->item($i)->getElementsByTagName('img')->length;
1028 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); 1029 $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
1029 } 1030 $input = $tagsList->item($i)->getElementsByTagName('input')->length;
1030 else if ( $this->getCharCount($tagsList->item($i), ',') < 10) { 1031 $a = $tagsList->item($i)->getElementsByTagName('a')->length;
1031 /** 1032
1032 * If there are not very many commas, and the number of 1033 $embedCount = 0;
1033 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element. 1034 $embeds = $tagsList->item($i)->getElementsByTagName('embed');
1034 **/ 1035 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1035 $p = $tagsList->item($i)->getElementsByTagName('p')->length; 1036 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1036 $img = $tagsList->item($i)->getElementsByTagName('img')->length; 1037 $embedCount++;
1037 $li = $tagsList->item($i)->getElementsByTagName('li')->length-100; 1038 }
1038 $input = $tagsList->item($i)->getElementsByTagName('input')->length; 1039 }
1039 1040 $embeds = $tagsList->item($i)->getElementsByTagName('iframe');
1040 $embedCount = 0; 1041 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1041 $embeds = $tagsList->item($i)->getElementsByTagName('embed'); 1042 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1042 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) { 1043 $embedCount++;
1043 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) { 1044 }
1044 $embedCount++; 1045 }
1045 } 1046
1046 } 1047 $linkDensity = $this->getLinkDensity($tagsList->item($i));
1047 1048 $contentLength = strlen($this->getInnerText($tagsList->item($i)));
1048 $linkDensity = $this->getLinkDensity($tagsList->item($i)); 1049 $toRemove = false;
1049 $contentLength = strlen($this->getInnerText($tagsList->item($i))); 1050
1050 $toRemove = false; 1051 if ($this->lightClean) {
1051 1052 $this->dbg('Light clean...');
1052 if ( $img > $p ) { 1053 if ( ($img > $p) && ($img > 4) ) {
1053 $toRemove = true; 1054 $this->dbg(' more than 4 images and more image elements than paragraph elements');
1054 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') { 1055 $toRemove = true;
1055 $toRemove = true; 1056 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1056 } else if ( $input > floor($p/3) ) { 1057 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1057 $toRemove = true; 1058 $toRemove = true;
1058 } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) { 1059 } else if ( $input > floor($p/3) ) {
1059 $toRemove = true; 1060 $this->dbg(' too many <input> elements');
1060 } else if($weight < 25 && $linkDensity > 0.2) { 1061 $toRemove = true;
1061 $toRemove = true; 1062 } else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
1062 } else if($weight >= 25 && $linkDensity > 0.5) { 1063 $this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');
1063 $toRemove = true; 1064 $toRemove = true;
1064 } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) { 1065 } else if($weight < 25 && $linkDensity > 0.2) {
1065 $toRemove = true; 1066 $this->dbg(' weight smaller than 25 and link density above 0.2');
1066 } 1067 $toRemove = true;
1067 1068 } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
1068 if ($toRemove) { 1069 $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
1069 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); 1070 $toRemove = true;
1070 } 1071 } else if($embedCount > 3) {
1071 } 1072 $this->dbg(' more than 3 embeds');
1072 } 1073 $toRemove = true;
1073 } 1074 }
1074 1075 } else {
1075 /** 1076 $this->dbg('Standard clean...');
1076 * Clean out spurious headers from an Element. Checks things like classnames and link density. 1077 if ( $img > $p ) {
1077 * 1078 $this->dbg(' more image elements than paragraph elements');
1078 * @param DOMElement $e 1079 $toRemove = true;
1079 * @return void 1080 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1080 */ 1081 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1081 public function cleanHeaders($e) { 1082 $toRemove = true;
1082 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) { 1083 } else if ( $input > floor($p/3) ) {
1083 $headers = $e->getElementsByTagName('h' . $headerIndex); 1084 $this->dbg(' too many <input> elements');
1084 for ($i=$headers->length-1; $i >=0; $i--) { 1085 $toRemove = true;
1085 if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) { 1086 } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
1086 $headers->item($i)->parentNode->removeChild($headers->item($i)); 1087 $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
1087 } 1088 $toRemove = true;
1088 } 1089 } else if($weight < 25 && $linkDensity > 0.2) {
1089 } 1090 $this->dbg(' weight smaller than 25 and link density above 0.2');
1090 } 1091 $toRemove = true;
1091 1092 } else if($weight >= 25 && $linkDensity > 0.5) {
1092 public function flagIsActive($flag) { 1093 $this->dbg(' weight above 25 but link density greater than 0.5');
1093 return ($this->flags & $flag) > 0; 1094 $toRemove = true;
1094 } 1095 } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
1095 1096 $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
1096 public function addFlag($flag) { 1097 $toRemove = true;
1097 $this->flags = $this->flags | $flag; 1098 }
1098 } 1099 }
1099 1100
1100 public function removeFlag($flag) { 1101 if ($toRemove) {
1101 $this->flags = $this->flags & ~$flag; 1102 //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
1102 } 1103 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1103} 1104 }
1104?> \ No newline at end of file 1105 }
1106 }
1107 }
1108
1109 /**
1110 * Clean out spurious headers from an Element. Checks things like classnames and link density.
1111 *
1112 * @param DOMElement $e
1113 * @return void
1114 */
1115 public function cleanHeaders($e) {
1116 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
1117 $headers = $e->getElementsByTagName('h' . $headerIndex);
1118 for ($i=$headers->length-1; $i >=0; $i--) {
1119 if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
1120 $headers->item($i)->parentNode->removeChild($headers->item($i));
1121 }
1122 }
1123 }
1124 }
1125
1126 public function flagIsActive($flag) {
1127 return ($this->flags & $flag) > 0;
1128 }
1129
1130 public function addFlag($flag) {
1131 $this->flags = $this->flags | $flag;
1132 }
1133
1134 public function removeFlag($flag) {
1135 $this->flags = $this->flags & ~$flag;
1136 }
1137} \ No newline at end of file