aboutsummaryrefslogtreecommitdiffhomepage
path: root/inc
diff options
context:
space:
mode:
authorNicolas Lœuillet <nicolas@loeuillet.org>2013-07-31 17:19:50 +0200
committerNicolas Lœuillet <nicolas@loeuillet.org>2013-07-31 17:19:50 +0200
commit85ebc80c7eaf88e4d57a52adb8e4c32d8cc34b64 (patch)
tree0012345c93b1fe8a242963bede0db370598d6f2b /inc
parent6588517b2d46cbc524260758d1fafd92b19ab681 (diff)
parent6d37de5f828b700843567a91418671919eed4be4 (diff)
downloadwallabag-d697309bc1b83b8315eae42748a6f906dab7b289.tar.gz
wallabag-d697309bc1b83b8315eae42748a6f906dab7b289.tar.zst
wallabag-d697309bc1b83b8315eae42748a6f906dab7b289.zip
Merge branch 'dev'0.3
Diffstat (limited to 'inc')
-rw-r--r--inc/JSLikeHTMLElement.php15
-rw-r--r--inc/Readability.php2164
-rw-r--r--inc/Session.class.php2
-rw-r--r--inc/config.php27
-rw-r--r--inc/functions.php82
-rw-r--r--inc/store/sqlite.class.php60
-rw-r--r--inc/store/store.class.php8
7 files changed, 1261 insertions, 1097 deletions
diff --git a/inc/JSLikeHTMLElement.php b/inc/JSLikeHTMLElement.php
index 0557205f..dfcc1be5 100644
--- a/inc/JSLikeHTMLElement.php
+++ b/inc/JSLikeHTMLElement.php
@@ -4,7 +4,7 @@
4* 4*
5* This class extends PHP's DOMElement to allow 5* This class extends PHP's DOMElement to allow
6* users to get and set the innerHTML property of 6* users to get and set the innerHTML property of
7* HTML elements in the same way it's done in 7* HTML elements in the same way it's done in
8* JavaScript. 8* JavaScript.
9* 9*
10* Example usage: 10* Example usage:
@@ -15,16 +15,16 @@
15* $doc->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); 15* $doc->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
16* $doc->loadHTML('<div><p>Para 1</p><p>Para 2</p></div>'); 16* $doc->loadHTML('<div><p>Para 1</p><p>Para 2</p></div>');
17* $elem = $doc->getElementsByTagName('div')->item(0); 17* $elem = $doc->getElementsByTagName('div')->item(0);
18* 18*
19* // print innerHTML 19* // print innerHTML
20* echo $elem->innerHTML; // prints '<p>Para 1</p><p>Para 2</p>' 20* echo $elem->innerHTML; // prints '<p>Para 1</p><p>Para 2</p>'
21* echo "\n\n"; 21* echo "\n\n";
22* 22*
23* // set innerHTML 23* // set innerHTML
24* $elem->innerHTML = '<a href="http://fivefilters.org">FiveFilters.org</a>'; 24* $elem->innerHTML = '<a href="http://fivefilters.org">FiveFilters.org</a>';
25* echo $elem->innerHTML; // prints '<a href="http://fivefilters.org">FiveFilters.org</a>' 25* echo $elem->innerHTML; // prints '<a href="http://fivefilters.org">FiveFilters.org</a>'
26* echo "\n\n"; 26* echo "\n\n";
27* 27*
28* // print document (with our changes) 28* // print document (with our changes)
29* echo $doc->saveXML(); 29* echo $doc->saveXML();
30* @endcode 30* @endcode
@@ -59,7 +59,7 @@ class JSLikeHTMLElement extends DOMElement
59 $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8'); 59 $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8');
60 // Using <htmlfragment> will generate a warning, but so will bad HTML 60 // Using <htmlfragment> will generate a warning, but so will bad HTML
61 // (and by this point, bad HTML is what we've got). 61 // (and by this point, bad HTML is what we've got).
62 // We use it (and suppress the warning) because an HTML fragment will 62 // We use it (and suppress the warning) because an HTML fragment will
63 // be wrapped around <html><body> tags which we don't really want to keep. 63 // be wrapped around <html><body> tags which we don't really want to keep.
64 // Note: despite the warning, if loadHTML succeeds it will return true. 64 // Note: despite the warning, if loadHTML succeeds it will return true.
65 $result = @$f->loadHTML('<htmlfragment>'.$value.'</htmlfragment>'); 65 $result = @$f->loadHTML('<htmlfragment>'.$value.'</htmlfragment>');
@@ -86,7 +86,7 @@ class JSLikeHTMLElement extends DOMElement
86 * @code 86 * @code
87 * $string = $div->innerHTML; 87 * $string = $div->innerHTML;
88 * @endcode 88 * @endcode
89 */ 89 */
90 public function __get($name) 90 public function __get($name)
91 { 91 {
92 if ($name == 'innerHTML') { 92 if ($name == 'innerHTML') {
@@ -106,5 +106,4 @@ class JSLikeHTMLElement extends DOMElement
106 { 106 {
107 return '['.$this->tagName.']'; 107 return '['.$this->tagName.']';
108 } 108 }
109} 109} \ No newline at end of file
110?> \ No newline at end of file
diff --git a/inc/Readability.php b/inc/Readability.php
index c50bf2ef..d28d28f9 100644
--- a/inc/Readability.php
+++ b/inc/Readability.php
@@ -1,7 +1,9 @@
1<?php 1<?php
2/** 2/**
3* Arc90's Readability ported to PHP for FiveFilters.org 3* Arc90's Readability ported to PHP for FiveFilters.org
4* Based on readability.js version 1.7.1 (without multi-page support) 4* Based on readability.js version 1.7.1 (without multi-page support)
5* Updated to allow HTML5 parsing with html5lib
6* Updated with lightClean mode to preserve more images and youtube/vimeo/viddler embeds
5* ------------------------------------------------------ 7* ------------------------------------------------------
6* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js 8* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
7* Arc90's project URL: http://lab.arc90.com/experiments/readability/ 9* Arc90's project URL: http://lab.arc90.com/experiments/readability/
@@ -10,35 +12,35 @@
10* More information: http://fivefilters.org/content-only/ 12* More information: http://fivefilters.org/content-only/
11* License: Apache License, Version 2.0 13* License: Apache License, Version 2.0
12* Requires: PHP5 14* Requires: PHP5
13* Date: 2011-07-22 15* Date: 2012-09-19
14* 16*
15* Differences between the PHP port and the original 17* Differences between the PHP port and the original
16* ------------------------------------------------------ 18* ------------------------------------------------------
17* Arc90's Readability is designed to run in the browser. It works on the DOM 19* Arc90's Readability is designed to run in the browser. It works on the DOM
18* tree (the parsed HTML) after the page's CSS styles have been applied and 20* tree (the parsed HTML) after the page's CSS styles have been applied and
19* Javascript code executed. This PHP port does not run inside a browser. 21* Javascript code executed. This PHP port does not run inside a browser.
20* We use PHP's ability to parse HTML to build our DOM tree, but we cannot 22* We use PHP's ability to parse HTML to build our DOM tree, but we cannot
21* rely on CSS or Javascript support. As such, the results will not always 23* rely on CSS or Javascript support. As such, the results will not always
22* match Arc90's Readability. (For example, if a web page contains CSS style 24* match Arc90's Readability. (For example, if a web page contains CSS style
23* rules or Javascript code which hide certain HTML elements from display, 25* rules or Javascript code which hide certain HTML elements from display,
24* Arc90's Readability will dismiss those from consideration but our PHP port, 26* Arc90's Readability will dismiss those from consideration but our PHP port,
25* unable to understand CSS or Javascript, will not know any better.) 27* unable to understand CSS or Javascript, will not know any better.)
26* 28*
27* Another significant difference is that the aim of Arc90's Readability is 29* Another significant difference is that the aim of Arc90's Readability is
28* to re-present the main content block of a given web page so users can 30* to re-present the main content block of a given web page so users can
29* read it more easily in their browsers. Correct identification, clean up, 31* read it more easily in their browsers. Correct identification, clean up,
30* and separation of the content block is only a part of this process. 32* and separation of the content block is only a part of this process.
31* This PHP port is only concerned with this part, it does not include code 33* This PHP port is only concerned with this part, it does not include code
32* that relates to presentation in the browser - Arc90 already do 34* that relates to presentation in the browser - Arc90 already do
33* that extremely well, and for PDF output there's FiveFilters.org's 35* that extremely well, and for PDF output there's FiveFilters.org's
34* PDF Newspaper: http://fivefilters.org/pdf-newspaper/. 36* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
35* 37*
36* Finally, this class contains methods that might be useful for developers 38* Finally, this class contains methods that might be useful for developers
37* working on HTML document fragments. So without deviating too much from 39* working on HTML document fragments. So without deviating too much from
38* the original code (which I don't want to do because it makes debugging 40* the original code (which I don't want to do because it makes debugging
39* and updating more difficult), I've tried to make it a little more 41* and updating more difficult), I've tried to make it a little more
40* developer friendly. You should be able to use the methods here on 42* developer friendly. You should be able to use the methods here on
41* existing DOMElement objects without passing an entire HTML document to 43* existing DOMElement objects without passing an entire HTML document to
42* be parsed. 44* be parsed.
43*/ 45*/
44 46
@@ -46,13 +48,13 @@
46require_once(dirname(__FILE__).'/JSLikeHTMLElement.php'); 48require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');
47 49
48// Alternative usage (for testing only!) 50// Alternative usage (for testing only!)
49// uncomment the lines below and call Readability.php in your browser 51// uncomment the lines below and call Readability.php in your browser
50// passing it the URL of the page you'd like content from, e.g.: 52// passing it the URL of the page you'd like content from, e.g.:
51// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php 53// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php
52 54
53/* 55/*
54if (!isset($_GET['url']) || $_GET['url'] == '') { 56if (!isset($_GET['url']) || $_GET['url'] == '') {
55 die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html'); 57 die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
56} 58}
57$url = $_GET['url']; 59$url = $_GET['url'];
58if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url; 60if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
@@ -62,1042 +64,1074 @@ $r->init();
62echo $r->articleContent->innerHTML; 64echo $r->articleContent->innerHTML;
63*/ 65*/
64 66
65
66class Readability 67class Readability
67{ 68{
68 /* constants */ 69 public $version = '1.7.1-without-multi-page';
69 const FLAG_STRIP_UNLIKELYS = 1; 70 public $convertLinksToFootnotes = false;
70 const FLAG_WEIGHT_CLASSES = 2; 71 public $revertForcedParagraphElements = true;
71 const FLAG_CLEAN_CONDITIONALLY = 4; 72 public $articleTitle;
72 73 public $articleContent;
73 public $version = '1.7.1-without-multi-page'; 74 public $dom;
74 public $convertLinksToFootnotes = false; 75 public $url = null; // optional - URL where HTML was retrieved
75 public $revertForcedParagraphElements = true; 76 public $debug = false;
76 public $articleTitle; 77 public $lightClean = true; // preserves more content (experimental) added 2012-09-19
77 public $articleContent; 78 protected $body = null; //
78 public $dom; 79 protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
79 public $url = null; // optional - URL where HTML was retrieved 80 protected $flags = 7; // 1 | 2 | 4; // Start with all flags set.
80 public $debug = false; 81 protected $success = false; // indicates whether we were able to extract or not
81 protected $body = null; // 82
82 protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later 83 /**
83 protected $flags = self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4; // Start with all flags set. 84 * All of the regular expressions in use within readability.
84 protected $success = false; // indicates whether we were able to extract or not 85 * Defined up here so we don't instantiate them repeatedly in loops.
85 86 **/
86 /** 87 public $regexps = array(
87 * All of the regular expressions in use within readability. 88 'unlikelyCandidates' => '/combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup/i',
88 * Defined up here so we don't instantiate them repeatedly in loops. 89 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
89 **/ 90 'positive' => '/article|body|content|entry|hentry|main|page|attachment|pagination|post|text|blog|story/i',
90 public $regexps = array( 91 'negative' => '/combx|comment|com-|contact|foot|footer|_nav|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
91 'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i', 92 'divToPElements' => '/<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i',
92 'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i', 93 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
93 'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i', 94 'replaceFonts' => '/<(\/?)font[^>]*>/i',
94 'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i', 95 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
95 'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i', 96 'normalize' => '/\s{2,}/',
96 'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i', 97 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
97 'replaceFonts' => '/<(\/?)font[^>]*>/i', 98 'video' => '!//(player\.|www\.)?(youtube|vimeo|viddler)\.com!i',
98 // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim() 99 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
99 'normalize' => '/\s{2,}/', 100 );
100 'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/', 101
101 'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i', 102 /* constants */
102 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i' 103 const FLAG_STRIP_UNLIKELYS = 1;
103 ); 104 const FLAG_WEIGHT_CLASSES = 2;
104 105 const FLAG_CLEAN_CONDITIONALLY = 4;
105 /** 106
106 * Create instance of Readability 107 /**
107 * @param string UTF-8 encoded string 108 * Create instance of Readability
108 * @param string (optional) URL associated with HTML (used for footnotes) 109 * @param string UTF-8 encoded string
109 */ 110 * @param string (optional) URL associated with HTML (used for footnotes)
110 function __construct($html, $url=null) 111 * @param string which parser to use for turning raw HTML into a DOMDocument (either 'libxml' or 'html5lib')
111 { 112 */
112 /* Turn all double br's into p's */ 113 function __construct($html, $url=null, $parser='libxml')
113 $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html); 114 {
114 $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); 115 $this->url = $url;
115 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); 116 /* Turn all double br's into p's */
116 $this->dom = new DOMDocument(); 117 $html = preg_replace($this->regexps['replaceBrs'], '</p><p>', $html);
117 $this->dom->preserveWhiteSpace = false; 118 $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html);
118 $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); 119 $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8");
119 if (trim($html) == '') $html = '<html></html>'; 120 if (trim($html) == '') $html = '<html></html>';
120 @$this->dom->loadHTML($html); 121 if ($parser=='html5lib' && ($this->dom = HTML5_Parser::parse($html))) {
121 $this->url = $url; 122 // all good
122 } 123 } else {
123 124 $this->dom = new DOMDocument();
124 /** 125 $this->dom->preserveWhiteSpace = false;
125 * Get article title element 126 @$this->dom->loadHTML($html);
126 * @return DOMElement 127 }
127 */ 128 $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement');
128 public function getTitle() { 129 }
129 return $this->articleTitle; 130
130 } 131 /**
131 132 * Get article title element
132 /** 133 * @return DOMElement
133 * Get article content element 134 */
134 * @return DOMElement 135 public function getTitle() {
135 */ 136 return $this->articleTitle;
136 public function getContent() { 137 }
137 return $this->articleContent; 138
138 } 139 /**
139 140 * Get article content element
140 /** 141 * @return DOMElement
141 * Runs readability. 142 */
142 * 143 public function getContent() {
143 * Workflow: 144 return $this->articleContent;
144 * 1. Prep the document by removing script tags, css, etc. 145 }
145 * 2. Build readability's DOM tree. 146
146 * 3. Grab the article content from the current dom tree. 147 /**
147 * 4. Replace the current DOM tree with the new one. 148 * Runs readability.
148 * 5. Read peacefully. 149 *
149 * 150 * Workflow:
150 * @return boolean true if we found content, false otherwise 151 * 1. Prep the document by removing script tags, css, etc.
151 **/ 152 * 2. Build readability's DOM tree.
152 public function init() 153 * 3. Grab the article content from the current dom tree.
153 { 154 * 4. Replace the current DOM tree with the new one.
154 if (!isset($this->dom->documentElement)) return false; 155 * 5. Read peacefully.
155 $this->removeScripts($this->dom); 156 *
156 //die($this->getInnerHTML($this->dom->documentElement)); 157 * @return boolean true if we found content, false otherwise
157 158 **/
158 // Assume successful outcome 159 public function init()
159 $this->success = true; 160 {
160 161 if (!isset($this->dom->documentElement)) return false;
161 $bodyElems = $this->dom->getElementsByTagName('body'); 162 $this->removeScripts($this->dom);
162 if ($bodyElems->length > 0) { 163 //die($this->getInnerHTML($this->dom->documentElement));
163 if ($this->bodyCache == null) { 164
164 $this->bodyCache = $bodyElems->item(0)->innerHTML; 165 // Assume successful outcome
165 } 166 $this->success = true;
166 if ($this->body == null) { 167
167 $this->body = $bodyElems->item(0); 168 $bodyElems = $this->dom->getElementsByTagName('body');
168 } 169 if ($bodyElems->length > 0) {
169 } 170 if ($this->bodyCache == null) {
170 171 $this->bodyCache = $bodyElems->item(0)->innerHTML;
171 $this->prepDocument(); 172 }
172 173 if ($this->body == null) {
173 //die($this->dom->documentElement->parentNode->nodeType); 174 $this->body = $bodyElems->item(0);
174 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); 175 }
175 //die($this->getInnerHTML($this->dom->documentElement)); 176 }
176 177
177 /* Build readability's DOM tree */ 178 $this->prepDocument();
178 $overlay = $this->dom->createElement('div'); 179
179 $innerDiv = $this->dom->createElement('div'); 180 //die($this->dom->documentElement->parentNode->nodeType);
180 $articleTitle = $this->getArticleTitle(); 181 //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement));
181 $articleContent = $this->grabArticle(); 182 //die($this->getInnerHTML($this->dom->documentElement));
182 183
183 if (!$articleContent) { 184 /* Build readability's DOM tree */
184 $this->success = false; 185 $overlay = $this->dom->createElement('div');
185 $articleContent = $this->dom->createElement('div'); 186 $innerDiv = $this->dom->createElement('div');
186 $articleContent->setAttribute('id', 'readability-content'); 187 $articleTitle = $this->getArticleTitle();
187 $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>'; 188 $articleContent = $this->grabArticle();
188 } 189
189 190 if (!$articleContent) {
190 $overlay->setAttribute('id', 'readOverlay'); 191 $this->success = false;
191 $innerDiv->setAttribute('id', 'readInner'); 192 $articleContent = $this->dom->createElement('div');
192 193 $articleContent->setAttribute('id', 'readability-content');
193 /* Glue the structure of our document together. */ 194 $articleContent->innerHTML = '<p>Sorry, Readability was unable to parse this page for content.</p>';
194 $innerDiv->appendChild($articleTitle); 195 }
195 $innerDiv->appendChild($articleContent); 196
196 $overlay->appendChild($innerDiv); 197 $overlay->setAttribute('id', 'readOverlay');
197 198 $innerDiv->setAttribute('id', 'readInner');
198 /* Clear the old HTML, insert the new content. */ 199
199 $this->body->innerHTML = ''; 200 /* Glue the structure of our document together. */
200 $this->body->appendChild($overlay); 201 $innerDiv->appendChild($articleTitle);
201 //document.body.insertBefore(overlay, document.body.firstChild); 202 $innerDiv->appendChild($articleContent);
202 $this->body->removeAttribute('style'); 203 $overlay->appendChild($innerDiv);
203 204
204 $this->postProcessContent($articleContent); 205 /* Clear the old HTML, insert the new content. */
205 206 $this->body->innerHTML = '';
206 // Set title and content instance variables 207 $this->body->appendChild($overlay);
207 $this->articleTitle = $articleTitle; 208 //document.body.insertBefore(overlay, document.body.firstChild);
208 $this->articleContent = $articleContent; 209 $this->body->removeAttribute('style');
209 210
210 return $this->success; 211 $this->postProcessContent($articleContent);
211 } 212
212 213 // Set title and content instance variables
213 /** 214 $this->articleTitle = $articleTitle;
214 * Debug 215 $this->articleContent = $articleContent;
215 */ 216
216 protected function dbg($msg) { 217 return $this->success;
217 if ($this->debug) echo '* ',$msg, '<br />', "\n"; 218 }
218 } 219
219 220 /**
220 /** 221 * Debug
221 * Run any post-process modifications to article content as necessary. 222 */
222 * 223 protected function dbg($msg) {
223 * @param DOMElement 224 if ($this->debug) echo '* ',$msg, "\n";
224 * @return void 225 }
225 */ 226
226 public function postProcessContent($articleContent) { 227 /**
227 if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) { 228 * Run any post-process modifications to article content as necessary.
228 $this->addFootnotes($articleContent); 229 *
229 } 230 * @param DOMElement
230 } 231 * @return void
231 232 */
232 /** 233 public function postProcessContent($articleContent) {
233 * Get the article title as an H1. 234 if ($this->convertLinksToFootnotes && !preg_match('/wikipedia\.org/', @$this->url)) {
234 * 235 $this->addFootnotes($articleContent);
235 * @return DOMElement 236 }
236 */ 237 }
237 protected function getArticleTitle() { 238
238 $curTitle = ''; 239 /**
239 $origTitle = ''; 240 * Get the article title as an H1.
240 241 *
241 try { 242 * @return DOMElement
242 $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0)); 243 */
243 } catch(Exception $e) {} 244 protected function getArticleTitle() {
244 245 $curTitle = '';
245 if (preg_match('/ [\|\-] /', $curTitle)) 246 $origTitle = '';
246 { 247
247 $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle); 248 try {
248 249 $curTitle = $origTitle = $this->getInnerText($this->dom->getElementsByTagName('title')->item(0));
249 if (count(explode(' ', $curTitle)) < 3) { 250 } catch(Exception $e) {}
250 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle); 251
251 } 252 if (preg_match('/ [\|\-] /', $curTitle))
252 } 253 {
253 else if (strpos($curTitle, ': ') !== false) 254 $curTitle = preg_replace('/(.*)[\|\-] .*/i', '$1', $origTitle);
254 { 255
255 $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle); 256 if (count(explode(' ', $curTitle)) < 3) {
256 257 $curTitle = preg_replace('/[^\|\-]*[\|\-](.*)/i', '$1', $origTitle);
257 if (count(explode(' ', $curTitle)) < 3) { 258 }
258 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle); 259 }
259 } 260 else if (strpos($curTitle, ': ') !== false)
260 } 261 {
261 else if(strlen($curTitle) > 150 || strlen($curTitle) < 15) 262 $curTitle = preg_replace('/.*:(.*)/i', '$1', $origTitle);
262 { 263
263 $hOnes = $this->dom->getElementsByTagName('h1'); 264 if (count(explode(' ', $curTitle)) < 3) {
264 if($hOnes->length == 1) 265 $curTitle = preg_replace('/[^:]*[:](.*)/i','$1', $origTitle);
265 { 266 }
266 $curTitle = $this->getInnerText($hOnes->item(0)); 267 }
267 } 268 else if(strlen($curTitle) > 150 || strlen($curTitle) < 15)
268 } 269 {
269 270 $hOnes = $this->dom->getElementsByTagName('h1');
270 $curTitle = trim($curTitle); 271 if($hOnes->length == 1)
271 272 {
272 if (count(explode(' ', $curTitle)) <= 4) { 273 $curTitle = $this->getInnerText($hOnes->item(0));
273 $curTitle = $origTitle; 274 }
274 } 275 }
275 276
276 $articleTitle = $this->dom->createElement('h1'); 277 $curTitle = trim($curTitle);
277 $articleTitle->innerHTML = $curTitle; 278
278 279 if (count(explode(' ', $curTitle)) <= 4) {
279 return $articleTitle; 280 $curTitle = $origTitle;
280 } 281 }
281 282
282 /** 283 $articleTitle = $this->dom->createElement('h1');
283 * Prepare the HTML document for readability to scrape it. 284 $articleTitle->innerHTML = $curTitle;
284 * This includes things like stripping javascript, CSS, and handling terrible markup. 285
285 * 286 return $articleTitle;
286 * @return void 287 }
287 **/ 288
288 protected function prepDocument() { 289 /**
289 /** 290 * Prepare the HTML document for readability to scrape it.
290 * In some cases a body element can't be found (if the HTML is totally hosed for example) 291 * This includes things like stripping javascript, CSS, and handling terrible markup.
291 * so we create a new body node and append it to the document. 292 *
292 */ 293 * @return void
293 if ($this->body == null) 294 **/
294 { 295 protected function prepDocument() {
295 $this->body = $this->dom->createElement('body'); 296 /**
296 $this->dom->documentElement->appendChild($this->body); 297 * In some cases a body element can't be found (if the HTML is totally hosed for example)
297 } 298 * so we create a new body node and append it to the document.
298 $this->body->setAttribute('id', 'readabilityBody'); 299 */
299 300 if ($this->body == null)
300 /* Remove all style tags in head */ 301 {
301 $styleTags = $this->dom->getElementsByTagName('style'); 302 $this->body = $this->dom->createElement('body');
302 for ($i = $styleTags->length-1; $i >= 0; $i--) 303 $this->dom->documentElement->appendChild($this->body);
303 { 304 }
304 $styleTags->item($i)->parentNode->removeChild($styleTags->item($i)); 305 $this->body->setAttribute('id', 'readabilityBody');
305 } 306
306 307 /* Remove all style tags in head */
307 /* Turn all double br's into p's */ 308 $styleTags = $this->dom->getElementsByTagName('style');
308 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */ 309 for ($i = $styleTags->length-1; $i >= 0; $i--)
309 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>'); 310 {
310 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. 311 $styleTags->item($i)->parentNode->removeChild($styleTags->item($i));
311 // Manipulating innerHTML as it's done in JS is not possible in PHP. 312 }
312 } 313
313 314 /* Turn all double br's into p's */
314 /** 315 /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
315 * For easier reading, convert this document to have footnotes at the bottom rather than inline links. 316 //document.body.innerHTML = document.body.innerHTML.replace(readability.regexps.replaceBrs, '</p><p>').replace(readability.regexps.replaceFonts, '<$1span>');
316 * @see http://www.roughtype.com/archives/2010/05/experiments_in.php 317 // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree.
317 * 318 // Manipulating innerHTML as it's done in JS is not possible in PHP.
318 * @return void 319 }
319 **/ 320
320 public function addFootnotes($articleContent) { 321 /**
321 $footnotesWrapper = $this->dom->createElement('div'); 322 * For easier reading, convert this document to have footnotes at the bottom rather than inline links.
322 $footnotesWrapper->setAttribute('id', 'readability-footnotes'); 323 * @see http://www.roughtype.com/archives/2010/05/experiments_in.php
323 $footnotesWrapper->innerHTML = '<h3>References</h3>'; 324 *
324 325 * @return void
325 $articleFootnotes = $this->dom->createElement('ol'); 326 **/
326 $articleFootnotes->setAttribute('id', 'readability-footnotes-list'); 327 public function addFootnotes($articleContent) {
327 $footnotesWrapper->appendChild($articleFootnotes); 328 $footnotesWrapper = $this->dom->createElement('div');
328 329 $footnotesWrapper->setAttribute('id', 'readability-footnotes');
329 $articleLinks = $articleContent->getElementsByTagName('a'); 330 $footnotesWrapper->innerHTML = '<h3>References</h3>';
330 331
331 $linkCount = 0; 332 $articleFootnotes = $this->dom->createElement('ol');
332 for ($i = 0; $i < $articleLinks->length; $i++) 333 $articleFootnotes->setAttribute('id', 'readability-footnotes-list');
333 { 334 $footnotesWrapper->appendChild($articleFootnotes);
334 $articleLink = $articleLinks->item($i); 335
335 $footnoteLink = $articleLink->cloneNode(true); 336 $articleLinks = $articleContent->getElementsByTagName('a');
336 $refLink = $this->dom->createElement('a'); 337
337 $footnote = $this->dom->createElement('li'); 338 $linkCount = 0;
338 $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST); 339 for ($i = 0; $i < $articleLinks->length; $i++)
339 if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST); 340 {
340 //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host, 341 $articleLink = $articleLinks->item($i);
341 $linkText = $this->getInnerText($articleLink); 342 $footnoteLink = $articleLink->cloneNode(true);
342 343 $refLink = $this->dom->createElement('a');
343 if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) { 344 $footnote = $this->dom->createElement('li');
344 continue; 345 $linkDomain = @parse_url($footnoteLink->getAttribute('href'), PHP_URL_HOST);
345 } 346 if (!$linkDomain && isset($this->url)) $linkDomain = @parse_url($this->url, PHP_URL_HOST);
346 347 //linkDomain = footnoteLink.host ? footnoteLink.host : document.location.host,
347 $linkCount++; 348 $linkText = $this->getInnerText($articleLink);
348 349
349 /** Add a superscript reference after the article link */ 350 if ((strpos($articleLink->getAttribute('class'), 'readability-DoNotFootnote') !== false) || preg_match($this->regexps['skipFootnoteLink'], $linkText)) {
350 $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount); 351 continue;
351 $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>'; 352 }
352 $refLink->setAttribute('class', 'readability-DoNotFootnote'); 353
353 $refLink->setAttribute('style', 'color: inherit;'); 354 $linkCount++;
354 355
355 //TODO: does this work or should we use DOMNode.isSameNode()? 356 /** Add a superscript reference after the article link */
356 if ($articleLink->parentNode->lastChild == $articleLink) { 357 $refLink->setAttribute('href', '#readabilityFootnoteLink-' . $linkCount);
357 $articleLink->parentNode->appendChild($refLink); 358 $refLink->innerHTML = '<small><sup>[' . $linkCount . ']</sup></small>';
358 } else { 359 $refLink->setAttribute('class', 'readability-DoNotFootnote');
359 $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling); 360 $refLink->setAttribute('style', 'color: inherit;');
360 } 361
361 362 //TODO: does this work or should we use DOMNode.isSameNode()?
362 $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;'); 363 if ($articleLink->parentNode->lastChild == $articleLink) {
363 $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount); 364 $articleLink->parentNode->appendChild($refLink);
364 365 } else {
365 $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> '; 366 $articleLink->parentNode->insertBefore($refLink, $articleLink->nextSibling);
366 367 }
367 $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText); 368
368 $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount); 369 $articleLink->setAttribute('style', 'color: inherit; text-decoration: none;');
369 370 $articleLink->setAttribute('name', 'readabilityLink-' . $linkCount);
370 $footnote->appendChild($footnoteLink); 371
371 if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>'; 372 $footnote->innerHTML = '<small><sup><a href="#readabilityLink-' . $linkCount . '" title="Jump to Link in Article">^</a></sup></small> ';
372 373
373 $articleFootnotes->appendChild($footnote); 374 $footnoteLink->innerHTML = ($footnoteLink->getAttribute('title') != '' ? $footnoteLink->getAttribute('title') : $linkText);
374 } 375 $footnoteLink->setAttribute('name', 'readabilityFootnoteLink-' . $linkCount);
375 376
376 if ($linkCount > 0) { 377 $footnote->appendChild($footnoteLink);
377 $articleContent->appendChild($footnotesWrapper); 378 if ($linkDomain) $footnote->innerHTML = $footnote->innerHTML . '<small> (' . $linkDomain . ')</small>';
378 } 379
379 } 380 $articleFootnotes->appendChild($footnote);
380 381 }
381 /** 382
382 * Reverts P elements with class 'readability-styled' 383 if ($linkCount > 0) {
383 * to text nodes - which is what they were before. 384 $articleContent->appendChild($footnotesWrapper);
384 * 385 }
385 * @param DOMElement 386 }
386 * @return void 387
387 */ 388 /**
388 function revertReadabilityStyledElements($articleContent) { 389 * Reverts P elements with class 'readability-styled'
389 $xpath = new DOMXPath($articleContent->ownerDocument); 390 * to text nodes - which is what they were before.
390 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent); 391 *
391 //$elems = $articleContent->getElementsByTagName('p'); 392 * @param DOMElement
392 for ($i = $elems->length-1; $i >= 0; $i--) { 393 * @return void
393 $e = $elems->item($i); 394 */
394 $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); 395 function revertReadabilityStyledElements($articleContent) {
395 //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') { 396 $xpath = new DOMXPath($articleContent->ownerDocument);
396 // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e); 397 $elems = $xpath->query('.//p[@class="readability-styled"]', $articleContent);
397 //} 398 //$elems = $articleContent->getElementsByTagName('p');
398 } 399 for ($i = $elems->length-1; $i >= 0; $i--) {
399 } 400 $e = $elems->item($i);
400 401 $e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e);
401 /** 402 //if ($e->hasAttribute('class') && $e->getAttribute('class') == 'readability-styled') {
402 * Prepare the article node for display. Clean out any inline styles, 403 // $e->parentNode->replaceChild($this->dom->createTextNode($e->textContent), $e);
403 * iframes, forms, strip extraneous <p> tags, etc. 404 //}
404 * 405 }
405 * @param DOMElement 406 }
406 * @return void 407
407 */ 408 /**
408 function prepArticle($articleContent) { 409 * Prepare the article node for display. Clean out any inline styles,
409 $this->cleanStyles($articleContent); 410 * iframes, forms, strip extraneous <p> tags, etc.
410 $this->killBreaks($articleContent); 411 *
411 if ($this->revertForcedParagraphElements) { 412 * @param DOMElement
412 $this->revertReadabilityStyledElements($articleContent); 413 * @return void
413 } 414 */
414 415 function prepArticle($articleContent) {
415 /* Clean out junk from the article content */ 416 $this->cleanStyles($articleContent);
416 $this->cleanConditionally($articleContent, 'form'); 417 $this->killBreaks($articleContent);
417 $this->clean($articleContent, 'object'); 418 if ($this->revertForcedParagraphElements) {
418 $this->clean($articleContent, 'h1'); 419 $this->revertReadabilityStyledElements($articleContent);
419 420 }
420 /** 421
421 * If there is only one h2, they are probably using it 422 /* Clean out junk from the article content */
422 * as a header and not a subheader, so remove it since we already have a header. 423 $this->cleanConditionally($articleContent, 'form');
423 ***/ 424 $this->clean($articleContent, 'object');
424 if ($articleContent->getElementsByTagName('h2')->length == 1) { 425 $this->clean($articleContent, 'h1');
425 $this->clean($articleContent, 'h2'); 426
426 } 427 /**
427 $this->clean($articleContent, 'iframe'); 428 * If there is only one h2, they are probably using it
428 429 * as a header and not a subheader, so remove it since we already have a header.
429 $this->cleanHeaders($articleContent); 430 ***/
430 431 if (!$this->lightClean && ($articleContent->getElementsByTagName('h2')->length == 1)) {
431 /* Do these last as the previous stuff may have removed junk that will affect these */ 432 $this->clean($articleContent, 'h2');
432 $this->cleanConditionally($articleContent, 'table'); 433 }
433 $this->cleanConditionally($articleContent, 'ul'); 434 $this->clean($articleContent, 'iframe');
434 $this->cleanConditionally($articleContent, 'div'); 435
435 436 $this->cleanHeaders($articleContent);
436 /* Remove extra paragraphs */ 437
437 $articleParagraphs = $articleContent->getElementsByTagName('p'); 438 /* Do these last as the previous stuff may have removed junk that will affect these */
438 for ($i = $articleParagraphs->length-1; $i >= 0; $i--) 439 $this->cleanConditionally($articleContent, 'table');
439 { 440 $this->cleanConditionally($articleContent, 'ul');
440 $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length; 441 $this->cleanConditionally($articleContent, 'div');
441 $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length; 442
442 $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length; 443 /* Remove extra paragraphs */
443 444 $articleParagraphs = $articleContent->getElementsByTagName('p');
444 if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '') 445 for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
445 { 446 {
446 $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i)); 447 $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
447 } 448 $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
448 } 449 $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
449 450 $iframeCount = $articleParagraphs->item($i)->getElementsByTagName('iframe')->length;
450 try { 451
451 $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML); 452 if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $iframeCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
452 //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p'); 453 {
453 } 454 $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
454 catch (Exception $e) { 455 }
455 $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e); 456 }
456 } 457
457 } 458 try {
458 459 $articleContent->innerHTML = preg_replace('/<br[^>]*>\s*<p/i', '<p', $articleContent->innerHTML);
459 /** 460 //articleContent.innerHTML = articleContent.innerHTML.replace(/<br[^>]*>\s*<p/gi, '<p');
460 * Initialize a node with the readability object. Also checks the 461 }
461 * className/id for special names to add to its score. 462 catch (Exception $e) {
462 * 463 $this->dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
463 * @param Element 464 }
464 * @return void 465 }
465 **/ 466
466 protected function initializeNode($node) { 467 /**
467 $readability = $this->dom->createAttribute('readability'); 468 * Initialize a node with the readability object. Also checks the
468 $readability->value = 0; // this is our contentScore 469 * className/id for special names to add to its score.
469 $node->setAttributeNode($readability); 470 *
470 471 * @param Element
471 switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case 472 * @return void
472 case 'DIV': 473 **/
473 $readability->value += 5; 474 protected function initializeNode($node) {
474 break; 475 $readability = $this->dom->createAttribute('readability');
475 476 $readability->value = 0; // this is our contentScore
476 case 'PRE': 477 $node->setAttributeNode($readability);
477 case 'TD': 478
478 case 'BLOCKQUOTE': 479 switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
479 $readability->value += 3; 480 case 'DIV':
480 break; 481 $readability->value += 5;
481 482 break;
482 case 'ADDRESS': 483
483 case 'OL': 484 case 'PRE':
484 case 'UL': 485 case 'TD':
485 case 'DL': 486 case 'BLOCKQUOTE':
486 case 'DD': 487 $readability->value += 3;
487 case 'DT': 488 break;
488 case 'LI': 489
489 case 'FORM': 490 case 'ADDRESS':
490 $readability->value -= 3; 491 case 'OL':
491 break; 492 case 'UL':
492 493 case 'DL':
493 case 'H1': 494 case 'DD':
494 case 'H2': 495 case 'DT':
495 case 'H3': 496 case 'LI':
496 case 'H4': 497 case 'FORM':
497 case 'H5': 498 $readability->value -= 3;
498 case 'H6': 499 break;
499 case 'TH': 500
500 $readability->value -= 5; 501 case 'H1':
501 break; 502 case 'H2':
502 } 503 case 'H3':
503 $readability->value += $this->getClassWeight($node); 504 case 'H4':
504 } 505 case 'H5':
505 506 case 'H6':
506 /*** 507 case 'TH':
507 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is 508 $readability->value -= 5;
508 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div. 509 break;
509 * 510 }
510 * @return DOMElement 511 $readability->value += $this->getClassWeight($node);
511 **/ 512 }
512 protected function grabArticle($page=null) { 513
513 $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS); 514 /***
514 if (!$page) $page = $this->dom; 515 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
515 $allElements = $page->getElementsByTagName('*'); 516 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
516 /** 517 *
517 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs 518 * @return DOMElement
518 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.) 519 **/
519 * 520 protected function grabArticle($page=null) {
520 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5 521 $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
521 * TODO: Shouldn't this be a reverse traversal? 522 if (!$page) $page = $this->dom;
522 **/ 523 $allElements = $page->getElementsByTagName('*');
523 $node = null; 524 /**
524 $nodesToScore = array(); 525 * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
525 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) { 526 * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
526 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) { 527 *
527 //$node = $targetList->item($nodeIndex); 528 * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
528 $tagName = strtoupper($node->tagName); 529 * TODO: Shouldn't this be a reverse traversal?
529 /* Remove unlikely candidates */ 530 **/
530 if ($stripUnlikelyCandidates) { 531 $node = null;
531 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id'); 532 $nodesToScore = array();
532 if ( 533 for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
533 preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) && 534 //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
534 !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) && 535 //$node = $targetList->item($nodeIndex);
535 $tagName != 'BODY' 536 $tagName = strtoupper($node->tagName);
536 ) 537 /* Remove unlikely candidates */
537 { 538 if ($stripUnlikelyCandidates) {
538 $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString); 539 $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
539 //$nodesToRemove[] = $node; 540 if (
540 $node->parentNode->removeChild($node); 541 preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
541 $nodeIndex--; 542 !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
542 continue; 543 $tagName != 'BODY'
543 } 544 )
544 } 545 {
545 546 $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
546 if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') { 547 //$nodesToRemove[] = $node;
547 $nodesToScore[] = $node; 548 $node->parentNode->removeChild($node);
548 } 549 $nodeIndex--;
549 550 continue;
550 /* Turn all divs that don't have children block level elements into p's */ 551 }
551 if ($tagName == 'DIV') { 552 }
552 if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) { 553
553 //$this->dbg('Altering div to p'); 554 if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
554 $newNode = $this->dom->createElement('p'); 555 $nodesToScore[] = $node;
555 try { 556 }
556 $newNode->innerHTML = $node->innerHTML; 557
557 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node); 558 /* Turn all divs that don't have children block level elements into p's */
558 $node->parentNode->replaceChild($newNode, $node); 559 if ($tagName == 'DIV') {
559 $nodeIndex--; 560 if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
560 $nodesToScore[] = $node; // or $newNode? 561 //$this->dbg('Altering div to p');
561 } 562 $newNode = $this->dom->createElement('p');
562 catch(Exception $e) { 563 try {
563 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e); 564 $newNode->innerHTML = $node->innerHTML;
564 } 565 //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
565 } 566 $node->parentNode->replaceChild($newNode, $node);
566 else 567 $nodeIndex--;
567 { 568 $nodesToScore[] = $node; // or $newNode?
568 // EXPERIMENTAL 569 }
569 // TODO: change these p elements back to text nodes after processing 570 catch(Exception $e) {
570 for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) { 571 $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
571 $childNode = $node->childNodes->item($i); 572 }
572 if ($childNode->nodeType == 3) { // XML_TEXT_NODE 573 }
573 //$this->dbg('replacing text node with a p tag with the same content.'); 574 else
574 $p = $this->dom->createElement('p'); 575 {
575 $p->innerHTML = $childNode->nodeValue; 576 /* EXPERIMENTAL */
576 $p->setAttribute('style', 'display: inline;'); 577 // TODO: change these p elements back to text nodes after processing
577 $p->setAttribute('class', 'readability-styled'); 578 for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
578 $childNode->parentNode->replaceChild($p, $childNode); 579 $childNode = $node->childNodes->item($i);
579 } 580 if ($childNode->nodeType == 3) { // XML_TEXT_NODE
580 } 581 //$this->dbg('replacing text node with a p tag with the same content.');
581 } 582 $p = $this->dom->createElement('p');
582 } 583 $p->innerHTML = $childNode->nodeValue;
583 } 584 $p->setAttribute('style', 'display: inline;');
584 585 $p->setAttribute('class', 'readability-styled');
585 /** 586 $childNode->parentNode->replaceChild($p, $childNode);
586 * Loop through all paragraphs, and assign a score to them based on how content-y they look. 587 }
587 * Then add their score to their parent node. 588 }
588 * 589 }
589 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density. 590 }
590 **/ 591 }
591 $candidates = array(); 592
592 for ($pt=0; $pt < count($nodesToScore); $pt++) { 593 /**
593 $parentNode = $nodesToScore[$pt]->parentNode; 594 * Loop through all paragraphs, and assign a score to them based on how content-y they look.
594 // $grandParentNode = $parentNode ? $parentNode->parentNode : null; 595 * Then add their score to their parent node.
595 $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null); 596 *
596 $innerText = $this->getInnerText($nodesToScore[$pt]); 597 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
597 598 **/
598 if (!$parentNode || !isset($parentNode->tagName)) { 599 $candidates = array();
599 continue; 600 for ($pt=0; $pt < count($nodesToScore); $pt++) {
600 } 601 $parentNode = $nodesToScore[$pt]->parentNode;
601 602 // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
602 /* If this paragraph is less than 25 characters, don't even count it. */ 603 $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
603 if(strlen($innerText) < 25) { 604 $innerText = $this->getInnerText($nodesToScore[$pt]);
604 continue; 605
605 } 606 if (!$parentNode || !isset($parentNode->tagName)) {
606 607 continue;
607 /* Initialize readability data for the parent. */ 608 }
608 if (!$parentNode->hasAttribute('readability')) 609
609 { 610 /* If this paragraph is less than 25 characters, don't even count it. */
610 $this->initializeNode($parentNode); 611 if(strlen($innerText) < 25) {
611 $candidates[] = $parentNode; 612 continue;
612 } 613 }
613 614
614 /* Initialize readability data for the grandparent. */ 615 /* Initialize readability data for the parent. */
615 if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName)) 616 if (!$parentNode->hasAttribute('readability'))
616 { 617 {
617 $this->initializeNode($grandParentNode); 618 $this->initializeNode($parentNode);
618 $candidates[] = $grandParentNode; 619 $candidates[] = $parentNode;
619 } 620 }
620 621
621 $contentScore = 0; 622 /* Initialize readability data for the grandparent. */
622 623 if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
623 /* Add a point for the paragraph itself as a base. */ 624 {
624 $contentScore++; 625 $this->initializeNode($grandParentNode);
625 626 $candidates[] = $grandParentNode;
626 /* Add points for any commas within this paragraph */ 627 }
627 $contentScore += count(explode(',', $innerText)); 628
628 629 $contentScore = 0;
629 /* For every 100 characters in this paragraph, add another point. Up to 3 points. */ 630
630 $contentScore += min(floor(strlen($innerText) / 100), 3); 631 /* Add a point for the paragraph itself as a base. */
631 632 $contentScore++;
632 /* Add the score to the parent. The grandparent gets half. */ 633
633 $parentNode->getAttributeNode('readability')->value += $contentScore; 634 /* Add points for any commas within this paragraph */
634 635 $contentScore += count(explode(',', $innerText));
635 if ($grandParentNode) { 636
636 $grandParentNode->getAttributeNode('readability')->value += $contentScore/2; 637 /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
637 } 638 $contentScore += min(floor(strlen($innerText) / 100), 3);
638 } 639
639 640 /* Add the score to the parent. The grandparent gets half. */
640 /** 641 $parentNode->getAttributeNode('readability')->value += $contentScore;
641 * After we've calculated scores, loop through all of the possible candidate nodes we found 642
642 * and find the one with the highest score. 643 if ($grandParentNode) {
643 **/ 644 $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
644 $topCandidate = null; 645 }
645 for ($c=0, $cl=count($candidates); $c < $cl; $c++) 646 }
646 { 647
647 /** 648 /**
648 * Scale the final candidates score based on link density. Good content should have a 649 * After we've calculated scores, loop through all of the possible candidate nodes we found
649 * relatively small link density (5% or less) and be mostly unaffected by this operation. 650 * and find the one with the highest score.
650 **/ 651 **/
651 $readability = $candidates[$c]->getAttributeNode('readability'); 652 $topCandidate = null;
652 $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c])); 653 for ($c=0, $cl=count($candidates); $c < $cl; $c++)
653 654 {
654 $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value); 655 /**
655 656 * Scale the final candidates score based on link density. Good content should have a
656 if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) { 657 * relatively small link density (5% or less) and be mostly unaffected by this operation.
657 $topCandidate = $candidates[$c]; 658 **/
658 } 659 $readability = $candidates[$c]->getAttributeNode('readability');
659 } 660 $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
660 661
661 /** 662 $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
662 * If we still have no top candidate, just use the body as a last resort. 663
663 * We also have to copy the body node so it is something we can modify. 664 if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
664 **/ 665 $topCandidate = $candidates[$c];
665 if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY') 666 }
666 { 667 }
667 $topCandidate = $this->dom->createElement('div'); 668
668 if ($page instanceof DOMDocument) { 669 /**
669 if (!isset($page->documentElement)) { 670 * If we still have no top candidate, just use the body as a last resort.
670 // we don't have a body either? what a mess! :) 671 * We also have to copy the body node so it is something we can modify.
671 } else { 672 **/
672 $topCandidate->innerHTML = $page->documentElement->innerHTML; 673 if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
673 $page->documentElement->innerHTML = ''; 674 {
674 $page->documentElement->appendChild($topCandidate); 675 $topCandidate = $this->dom->createElement('div');
675 } 676 if ($page instanceof DOMDocument) {
676 } else { 677 if (!isset($page->documentElement)) {
677 $topCandidate->innerHTML = $page->innerHTML; 678 // we don't have a body either? what a mess! :)
678 $page->innerHTML = ''; 679 } else {
679 $page->appendChild($topCandidate); 680 $topCandidate->innerHTML = $page->documentElement->innerHTML;
680 } 681 $page->documentElement->innerHTML = '';
681 $this->initializeNode($topCandidate); 682 $page->documentElement->appendChild($topCandidate);
682 } 683 }
683 684 } else {
684 /** 685 $topCandidate->innerHTML = $page->innerHTML;
685 * Now that we have the top candidate, look through its siblings for content that might also be related. 686 $page->innerHTML = '';
686 * Things like preambles, content split by ads that we removed, etc. 687 $page->appendChild($topCandidate);
687 **/ 688 }
688 $articleContent = $this->dom->createElement('div'); 689 $this->initializeNode($topCandidate);
689 $articleContent->setAttribute('id', 'readability-content'); 690 }
690 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2); 691
691 $siblingNodes = $topCandidate->parentNode->childNodes; 692 /**
692 if (!isset($siblingNodes)) { 693 * Now that we have the top candidate, look through its siblings for content that might also be related.
693 $siblingNodes = new stdClass; 694 * Things like preambles, content split by ads that we removed, etc.
694 $siblingNodes->length = 0; 695 **/
695 } 696 $articleContent = $this->dom->createElement('div');
696 697 $articleContent->setAttribute('id', 'readability-content');
697 for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++) 698 $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
698 { 699 $siblingNodes = $topCandidate->parentNode->childNodes;
699 $siblingNode = $siblingNodes->item($s); 700 if (!isset($siblingNodes)) {
700 $append = false; 701 $siblingNodes = new stdClass;
701 702 $siblingNodes->length = 0;
702 $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : '')); 703 }
703 704
704 //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown')); 705 for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
705 706 {
706 if ($siblingNode === $topCandidate) 707 $siblingNode = $siblingNodes->item($s);
707 // or if ($siblingNode->isSameNode($topCandidate)) 708 $append = false;
708 { 709
709 $append = true; 710 $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
710 } 711
711 712 //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
712 $contentBonus = 0; 713
713 /* Give a bonus if sibling nodes and top candidates have the example same classname */ 714 if ($siblingNode === $topCandidate)
714 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') { 715 // or if ($siblingNode->isSameNode($topCandidate))
715 $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2; 716 {
716 } 717 $append = true;
717 718 }
718 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold) 719
719 { 720 $contentBonus = 0;
720 $append = true; 721 /* Give a bonus if sibling nodes and top candidates have the example same classname */
721 } 722 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
722 723 $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
723 if (strtoupper($siblingNode->nodeName) == 'P') { 724 }
724 $linkDensity = $this->getLinkDensity($siblingNode); 725
725 $nodeContent = $this->getInnerText($siblingNode); 726 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
726 $nodeLength = strlen($nodeContent); 727 {
727 728 $append = true;
728 if ($nodeLength > 80 && $linkDensity < 0.25) 729 }
729 { 730
730 $append = true; 731 if (strtoupper($siblingNode->nodeName) == 'P') {
731 } 732 $linkDensity = $this->getLinkDensity($siblingNode);
732 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent)) 733 $nodeContent = $this->getInnerText($siblingNode);
733 { 734 $nodeLength = strlen($nodeContent);
734 $append = true; 735
735 } 736 if ($nodeLength > 80 && $linkDensity < 0.25)
736 } 737 {
737 738 $append = true;
738 /* Look for a special classname */ 739 }
739 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('class') && $siblingNode->getAttribute('class') != '') 740 else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
740 { 741 {
741 if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('class'))) { 742 $append = true;
742 $append = true; 743 }
743 } 744 }
744 } 745
745 746 if ($append)
746 /* Look for a special classname */ 747 {
747 if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('id') && $siblingNode->getAttribute('id') != '') 748 $this->dbg('Appending node: ' . $siblingNode->nodeName);
748 { 749
749 if (preg_match($this->regexps['okMaybeItsACandidate'], $siblingNode->getAttribute('id'))) { 750 $nodeToAppend = null;
750 $append = true; 751 $sibNodeName = strtoupper($siblingNode->nodeName);
751 } 752 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
752 } 753 /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
753 754
754 755 $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
755 if ($append) 756 $nodeToAppend = $this->dom->createElement('div');
756 { 757 try {
757 $this->dbg('Appending node: ' . $siblingNode->nodeName); 758 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
758 759 $nodeToAppend->innerHTML = $siblingNode->innerHTML;
759 $nodeToAppend = null; 760 }
760 $sibNodeName = strtoupper($siblingNode->nodeName); 761 catch(Exception $e)
761 if ($sibNodeName != 'DIV' && $sibNodeName != 'P') { 762 {
762 /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */ 763 $this->dbg('Could not alter siblingNode to div, reverting back to original.');
763 764 $nodeToAppend = $siblingNode;
764 $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.'); 765 $s--;
765 $nodeToAppend = $this->dom->createElement('div'); 766 $sl--;
766 try { 767 }
767 $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id')); 768 } else {
768 $nodeToAppend->innerHTML = $siblingNode->innerHTML; 769 $nodeToAppend = $siblingNode;
769 } 770 $s--;
770 catch(Exception $e) 771 $sl--;
771 { 772 }
772 $this->dbg('Could not alter siblingNode to div, reverting back to original.'); 773
773 $nodeToAppend = $siblingNode; 774 /* To ensure a node does not interfere with readability styles, remove its classnames */
774 $s--; 775 $nodeToAppend->removeAttribute('class');
775 $sl--; 776
776 } 777 /* Append sibling and subtract from our list because it removes the node when you append to another node */
777 } else { 778 $articleContent->appendChild($nodeToAppend);
778 $nodeToAppend = $siblingNode; 779 }
779 $s--; 780 }
780 $sl--; 781
781 } 782 /**
782 783 * So we have all of the content that we need. Now we clean it up for presentation.
783 /* To ensure a node does not interfere with readability styles, remove its classnames */ 784 **/
784 $nodeToAppend->removeAttribute('class'); 785 $this->prepArticle($articleContent);
785 786
786 /* Append sibling and subtract from our list because it removes the node when you append to another node */ 787 /**
787 $articleContent->appendChild($nodeToAppend); 788 * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
788 } 789 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
789 } 790 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
790 791 * finding the -right- content.
791 /** 792 **/
792 * So we have all of the content that we need. Now we clean it up for presentation. 793 if (strlen($this->getInnerText($articleContent, false)) < 250)
793 **/ 794 {
794 $this->prepArticle($articleContent); 795 // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7
795 796 // in the meantime, we check and create an empty element if it's not there.
796 /** 797 if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body');
797 * Now that we've gone through the full algorithm, check to see if we got any meaningful content. 798 $this->body->innerHTML = $this->bodyCache;
798 * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher 799
799 * likelihood of finding the content, and the sieve approach gives us a higher likelihood of 800 if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
800 * finding the -right- content. 801 $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
801 **/ 802 return $this->grabArticle($this->body);
802 if (strlen($this->getInnerText($articleContent, false)) < 250) 803 }
803 { 804 else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
804 // TODO: find out why element disappears sometimes, e.g. for this URL http://www.businessinsider.com/6-hedge-fund-etfs-for-average-investors-2011-7 805 $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
805 // in the meantime, we check and create an empty element if it's not there. 806 return $this->grabArticle($this->body);
806 if (!isset($this->body->childNodes)) $this->body = $this->dom->createElement('body'); 807 }
807 $this->body->innerHTML = $this->bodyCache; 808 else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
808 809 $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
809 if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) { 810 return $this->grabArticle($this->body);
810 $this->removeFlag(self::FLAG_STRIP_UNLIKELYS); 811 }
811 return $this->grabArticle($this->body); 812 else {
812 } 813 return false;
813 else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { 814 }
814 $this->removeFlag(self::FLAG_WEIGHT_CLASSES); 815 }
815 return $this->grabArticle($this->body); 816 return $articleContent;
816 } 817 }
817 else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { 818
818 $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY); 819 /**
819 return $this->grabArticle($this->body); 820 * Remove script tags from document
820 } 821 *
821 else { 822 * @param DOMElement
822 return false; 823 * @return void
823 } 824 */
824 } 825 public function removeScripts($doc) {
825 return $articleContent; 826 $scripts = $doc->getElementsByTagName('script');
826 } 827 for($i = $scripts->length-1; $i >= 0; $i--)
827 828 {
828 /** 829 $scripts->item($i)->parentNode->removeChild($scripts->item($i));
829 * Remove script tags from document 830 }
830 * 831 }
831 * @param DOMElement 832
832 * @return void 833 /**
833 */ 834 * Get the inner text of a node.
834 public function removeScripts($doc) { 835 * This also strips out any excess whitespace to be found.
835 $scripts = $doc->getElementsByTagName('script'); 836 *
836 for($i = $scripts->length-1; $i >= 0; $i--) 837 * @param DOMElement $
837 { 838 * @param boolean $normalizeSpaces (default: true)
838 $scripts->item($i)->parentNode->removeChild($scripts->item($i)); 839 * @return string
839 } 840 **/
840 } 841 public function getInnerText($e, $normalizeSpaces=true) {
841 842 $textContent = '';
842 /** 843
843 * Get the inner text of a node. 844 if (!isset($e->textContent) || $e->textContent == '') {
844 * This also strips out any excess whitespace to be found. 845 return '';
845 * 846 }
846 * @param DOMElement $ 847
847 * @param boolean $normalizeSpaces (default: true) 848 $textContent = trim($e->textContent);
848 * @return string 849
849 **/ 850 if ($normalizeSpaces) {
850 public function getInnerText($e, $normalizeSpaces=true) { 851 return preg_replace($this->regexps['normalize'], ' ', $textContent);
851 $textContent = ''; 852 } else {
852 853 return $textContent;
853 if (!isset($e->textContent) || $e->textContent == '') { 854 }
854 return ''; 855 }
855 } 856
856 857 /**
857 $textContent = trim($e->textContent); 858 * Get the number of times a string $s appears in the node $e.
858 859 *
859 if ($normalizeSpaces) { 860 * @param DOMElement $e
860 return preg_replace($this->regexps['normalize'], ' ', $textContent); 861 * @param string - what to count. Default is ","
861 } else { 862 * @return number (integer)
862 return $textContent; 863 **/
863 } 864 public function getCharCount($e, $s=',') {
864 } 865 return substr_count($this->getInnerText($e), $s);
865 866 }
866 /** 867
867 * Get the number of times a string $s appears in the node $e. 868 /**
868 * 869 * Remove the style attribute on every $e and under.
869 * @param DOMElement $e 870 *
870 * @param string - what to count. Default is "," 871 * @param DOMElement $e
871 * @return number (integer) 872 * @return void
872 **/ 873 */
873 public function getCharCount($e, $s=',') { 874 public function cleanStyles($e) {
874 return substr_count($this->getInnerText($e), $s); 875 if (!is_object($e)) return;
875 } 876 $elems = $e->getElementsByTagName('*');
876 877 foreach ($elems as $elem) {
877 /** 878 $elem->removeAttribute('style');
878 * Remove the style attribute on every $e and under. 879 }
879 * 880 }
880 * @param DOMElement $e 881
881 * @return void 882 /**
882 */ 883 * Get the density of links as a percentage of the content
883 public function cleanStyles($e) { 884 * This is the amount of text that is inside a link divided by the total text in the node.
884 if (!is_object($e)) return; 885 *
885 $elems = $e->getElementsByTagName('*'); 886 * @param DOMElement $e
886 foreach ($elems as $elem) { 887 * @return number (float)
887 $elem->removeAttribute('style'); 888 */
888 } 889 public function getLinkDensity($e) {
889 } 890 $links = $e->getElementsByTagName('a');
890 891 $textLength = strlen($this->getInnerText($e));
891 /** 892 $linkLength = 0;
892 * Get the density of links as a percentage of the content 893 for ($i=0, $il=$links->length; $i < $il; $i++)
893 * This is the amount of text that is inside a link divided by the total text in the node. 894 {
894 * 895 $linkLength += strlen($this->getInnerText($links->item($i)));
895 * @param DOMElement $e 896 }
896 * @return number (float) 897 if ($textLength > 0) {
897 */ 898 return $linkLength / $textLength;
898 public function getLinkDensity($e) { 899 } else {
899 $links = $e->getElementsByTagName('a'); 900 return 0;
900 $textLength = strlen($this->getInnerText($e)); 901 }
901 $linkLength = 0; 902 }
902 for ($i=0, $il=$links->length; $i < $il; $i++) 903
903 { 904 /**
904 $linkLength += strlen($this->getInnerText($links->item($i))); 905 * Get an elements class/id weight. Uses regular expressions to tell if this
905 } 906 * element looks good or bad.
906 if ($textLength > 0) { 907 *
907 return $linkLength / $textLength; 908 * @param DOMElement $e
908 } else { 909 * @return number (Integer)
909 return 0; 910 */
910 } 911 public function getClassWeight($e) {
911 } 912 if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
912 913 return 0;
913 /** 914 }
914 * Get an elements class/id weight. Uses regular expressions to tell if this 915
915 * element looks good or bad. 916 $weight = 0;
916 * 917
917 * @param DOMElement $e 918 /* Look for a special classname */
918 * @return number (Integer) 919 if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
919 */ 920 {
920 public function getClassWeight($e) { 921 if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
921 if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) { 922 $weight -= 25;
922 return 0; 923 }
923 } 924 if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
924 925 $weight += 25;
925 $weight = 0; 926 }
926 927 }
927 /* Look for a special classname */ 928
928 if ($e->hasAttribute('class') && $e->getAttribute('class') != '') 929 /* Look for a special ID */
929 { 930 if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
930 if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) { 931 {
931 $weight -= 25; 932 if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
932 } 933 $weight -= 25;
933 if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) { 934 }
934 $weight += 25; 935 if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
935 } 936 $weight += 25;
936 } 937 }
937 938 }
938 /* Look for a special ID */ 939 return $weight;
939 if ($e->hasAttribute('id') && $e->getAttribute('id') != '') 940 }
940 { 941
941 if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) { 942 /**
942 $weight -= 25; 943 * Remove extraneous break tags from a node.
943 } 944 *
944 if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) { 945 * @param DOMElement $node
945 $weight += 25; 946 * @return void
946 } 947 */
947 } 948 public function killBreaks($node) {
948 return $weight; 949 $html = $node->innerHTML;
949 } 950 $html = preg_replace($this->regexps['killBreaks'], '<br />', $html);
950 951 $node->innerHTML = $html;
951 /** 952 }
952 * Remove extraneous break tags from a node. 953
953 * 954 /**
954 * @param DOMElement $node 955 * Clean a node of all elements of type "tag".
955 * @return void 956 * (Unless it's a youtube/vimeo video. People love movies.)
956 */ 957 *
957 public function killBreaks($node) { 958 * Updated 2012-09-18 to preserve youtube/vimeo iframes
958 $html = $node->innerHTML; 959 *
959 $html = preg_replace($this->regexps['killBreaks'], '<br />', $html); 960 * @param DOMElement $e
960 $node->innerHTML = $html; 961 * @param string $tag
961 } 962 * @return void
962 963 */
963 /** 964 public function clean($e, $tag) {
964 * Clean a node of all elements of type "tag". 965 $targetList = $e->getElementsByTagName($tag);
965 * (Unless it's a youtube/vimeo video. People love movies.) 966 $isEmbed = ($tag == 'iframe' || $tag == 'object' || $tag == 'embed');
966 * 967
967 * @param DOMElement $e 968 for ($y=$targetList->length-1; $y >= 0; $y--) {
968 * @param string $tag 969 /* Allow youtube and vimeo videos through as people usually want to see those. */
969 * @return void 970 if ($isEmbed) {
970 */ 971 $attributeValues = '';
971 public function clean($e, $tag) { 972 for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
972 $targetList = $e->getElementsByTagName($tag); 973 $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
973 $isEmbed = ($tag == 'object' || $tag == 'embed'); 974 }
974 975
975 for ($y=$targetList->length-1; $y >= 0; $y--) { 976 /* First, check the elements attributes to see if any of them contain youtube or vimeo */
976 /* Allow youtube and vimeo videos through as people usually want to see those. */ 977 if (preg_match($this->regexps['video'], $attributeValues)) {
977 if ($isEmbed) { 978 continue;
978 $attributeValues = ''; 979 }
979 for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) { 980
980 $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test) 981 /* Then check the elements inside this element for the same. */
981 } 982 if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
982 983 continue;
983 /* First, check the elements attributes to see if any of them contain youtube or vimeo */ 984 }
984 if (preg_match($this->regexps['video'], $attributeValues)) { 985 }
985 continue; 986 $targetList->item($y)->parentNode->removeChild($targetList->item($y));
986 } 987 }
987 988 }
988 /* Then check the elements inside this element for the same. */ 989
989 if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) { 990 /**
990 continue; 991 * Clean an element of all tags of type "tag" if they look fishy.
991 } 992 * "Fishy" is an algorithm based on content length, classnames,
992 } 993 * link density, number of images & embeds, etc.
993 $targetList->item($y)->parentNode->removeChild($targetList->item($y)); 994 *
994 } 995 * @param DOMElement $e
995 } 996 * @param string $tag
996 997 * @return void
997 /** 998 */
998 * Clean an element of all tags of type "tag" if they look fishy. 999 public function cleanConditionally($e, $tag) {
999 * "Fishy" is an algorithm based on content length, classnames, 1000 if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
1000 * link density, number of images & embeds, etc. 1001 return;
1001 * 1002 }
1002 * @param DOMElement $e 1003
1003 * @param string $tag 1004 $tagsList = $e->getElementsByTagName($tag);
1004 * @return void 1005 $curTagsLength = $tagsList->length;
1005 */ 1006
1006 public function cleanConditionally($e, $tag) { 1007 /**
1007 if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) { 1008 * Gather counts for other typical elements embedded within.
1008 return; 1009 * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
1009 } 1010 *
1010 1011 * TODO: Consider taking into account original contentScore here.
1011 $tagsList = $e->getElementsByTagName($tag); 1012 */
1012 $curTagsLength = $tagsList->length; 1013 for ($i=$curTagsLength-1; $i >= 0; $i--) {
1013 1014 $weight = $this->getClassWeight($tagsList->item($i));
1014 /** 1015 $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
1015 * Gather counts for other typical elements embedded within. 1016
1016 * Traverse backwards so we can remove nodes at the same time without effecting the traversal. 1017 $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
1017 * 1018
1018 * TODO: Consider taking into account original contentScore here. 1019 if ($weight + $contentScore < 0) {
1019 */ 1020 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1020 for ($i=$curTagsLength-1; $i >= 0; $i--) { 1021 }
1021 $weight = $this->getClassWeight($tagsList->item($i)); 1022 else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
1022 $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0; 1023 /**
1023 1024 * If there are not very many commas, and the number of
1024 $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : '')); 1025 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
1025 1026 **/
1026 if ($weight + $contentScore < 0) { 1027 $p = $tagsList->item($i)->getElementsByTagName('p')->length;
1027 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); 1028 $img = $tagsList->item($i)->getElementsByTagName('img')->length;
1028 } 1029 $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
1029 else if ( $this->getCharCount($tagsList->item($i), ',') < 10) { 1030 $input = $tagsList->item($i)->getElementsByTagName('input')->length;
1030 /** 1031 $a = $tagsList->item($i)->getElementsByTagName('a')->length;
1031 * If there are not very many commas, and the number of 1032
1032 * non-paragraph elements is more than paragraphs or other ominous signs, remove the element. 1033 $embedCount = 0;
1033 **/ 1034 $embeds = $tagsList->item($i)->getElementsByTagName('embed');
1034 $p = $tagsList->item($i)->getElementsByTagName('p')->length; 1035 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1035 $img = $tagsList->item($i)->getElementsByTagName('img')->length; 1036 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1036 $li = $tagsList->item($i)->getElementsByTagName('li')->length-100; 1037 $embedCount++;
1037 $input = $tagsList->item($i)->getElementsByTagName('input')->length; 1038 }
1038 1039 }
1039 $embedCount = 0; 1040 $embeds = $tagsList->item($i)->getElementsByTagName('iframe');
1040 $embeds = $tagsList->item($i)->getElementsByTagName('embed'); 1041 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1041 for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) { 1042 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1042 if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) { 1043 $embedCount++;
1043 $embedCount++; 1044 }
1044 } 1045 }
1045 } 1046
1046 1047 $linkDensity = $this->getLinkDensity($tagsList->item($i));
1047 $linkDensity = $this->getLinkDensity($tagsList->item($i)); 1048 $contentLength = strlen($this->getInnerText($tagsList->item($i)));
1048 $contentLength = strlen($this->getInnerText($tagsList->item($i))); 1049 $toRemove = false;
1049 $toRemove = false; 1050
1050 1051 if ($this->lightClean) {
1051 if ( $img > $p ) { 1052 $this->dbg('Light clean...');
1052 $toRemove = true; 1053 if ( ($img > $p) && ($img > 4) ) {
1053 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') { 1054 $this->dbg(' more than 4 images and more image elements than paragraph elements');
1054 $toRemove = true; 1055 $toRemove = true;
1055 } else if ( $input > floor($p/3) ) { 1056 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1056 $toRemove = true; 1057 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1057 } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) { 1058 $toRemove = true;
1058 $toRemove = true; 1059 } else if ( $input > floor($p/3) ) {
1059 } else if($weight < 25 && $linkDensity > 0.2) { 1060 $this->dbg(' too many <input> elements');
1060 $toRemove = true; 1061 $toRemove = true;
1061 } else if($weight >= 25 && $linkDensity > 0.5) { 1062 } else if ($contentLength < 25 && ($embedCount === 0 && ($img === 0 || $img > 2))) {
1062 $toRemove = true; 1063 $this->dbg(' content length less than 25 chars, 0 embeds and either 0 images or more than 2 images');
1063 } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) { 1064 $toRemove = true;
1064 $toRemove = true; 1065 } else if($weight < 25 && $linkDensity > 0.2) {
1065 } 1066 $this->dbg(' weight smaller than 25 and link density above 0.2');
1066 1067 $toRemove = true;
1067 if ($toRemove) { 1068 } else if($a > 2 && ($weight >= 25 && $linkDensity > 0.5)) {
1068 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i)); 1069 $this->dbg(' more than 2 links and weight above 25 but link density greater than 0.5');
1069 } 1070 $toRemove = true;
1070 } 1071 } else if($embedCount > 3) {
1071 } 1072 $this->dbg(' more than 3 embeds');
1072 } 1073 $toRemove = true;
1073 1074 }
1074 /** 1075 } else {
1075 * Clean out spurious headers from an Element. Checks things like classnames and link density. 1076 $this->dbg('Standard clean...');
1076 * 1077 if ( $img > $p ) {
1077 * @param DOMElement $e 1078 $this->dbg(' more image elements than paragraph elements');
1078 * @return void 1079 $toRemove = true;
1079 */ 1080 } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1080 public function cleanHeaders($e) { 1081 $this->dbg(' too many <li> elements, and parent is not <ul> or <ol>');
1081 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) { 1082 $toRemove = true;
1082 $headers = $e->getElementsByTagName('h' . $headerIndex); 1083 } else if ( $input > floor($p/3) ) {
1083 for ($i=$headers->length-1; $i >=0; $i--) { 1084 $this->dbg(' too many <input> elements');
1084 if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) { 1085 $toRemove = true;
1085 $headers->item($i)->parentNode->removeChild($headers->item($i)); 1086 } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
1086 } 1087 $this->dbg(' content length less than 25 chars and 0 images, or more than 2 images');
1087 } 1088 $toRemove = true;
1088 } 1089 } else if($weight < 25 && $linkDensity > 0.2) {
1089 } 1090 $this->dbg(' weight smaller than 25 and link density above 0.2');
1090 1091 $toRemove = true;
1091 public function flagIsActive($flag) { 1092 } else if($weight >= 25 && $linkDensity > 0.5) {
1092 return ($this->flags & $flag) > 0; 1093 $this->dbg(' weight above 25 but link density greater than 0.5');
1093 } 1094 $toRemove = true;
1094 1095 } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
1095 public function addFlag($flag) { 1096 $this->dbg(' 1 embed and content length smaller than 75 chars, or more than one embed');
1096 $this->flags = $this->flags | $flag; 1097 $toRemove = true;
1097 } 1098 }
1098 1099 }
1099 public function removeFlag($flag) { 1100
1100 $this->flags = $this->flags & ~$flag; 1101 if ($toRemove) {
1101 } 1102 //$this->dbg('Removing: '.$tagsList->item($i)->innerHTML);
1102} 1103 $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1103?> \ No newline at end of file 1104 }
1105 }
1106 }
1107 }
1108
1109 /**
1110 * Clean out spurious headers from an Element. Checks things like classnames and link density.
1111 *
1112 * @param DOMElement $e
1113 * @return void
1114 */
1115 public function cleanHeaders($e) {
1116 for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
1117 $headers = $e->getElementsByTagName('h' . $headerIndex);
1118 for ($i=$headers->length-1; $i >=0; $i--) {
1119 if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
1120 $headers->item($i)->parentNode->removeChild($headers->item($i));
1121 }
1122 }
1123 }
1124 }
1125
1126 public function flagIsActive($flag) {
1127 return ($this->flags & $flag) > 0;
1128 }
1129
1130 public function addFlag($flag) {
1131 $this->flags = $this->flags | $flag;
1132 }
1133
1134 public function removeFlag($flag) {
1135 $this->flags = $this->flags & ~$flag;
1136 }
1137} \ No newline at end of file
diff --git a/inc/Session.class.php b/inc/Session.class.php
index ee12b3d1..eff924cc 100644
--- a/inc/Session.class.php
+++ b/inc/Session.class.php
@@ -93,7 +93,7 @@ class Session
93 // Force logout 93 // Force logout
94 public static function logout() 94 public static function logout()
95 { 95 {
96 unset($_SESSION['uid'],$_SESSION['info'],$_SESSION['expires_on'],$_SESSION['tokens']); 96 unset($_SESSION['uid'],$_SESSION['info'],$_SESSION['expires_on'],$_SESSION['tokens'], $_SESSION['login'], $_SESSION['pass']);
97 } 97 }
98 98
99 // Make sure user is logged in. 99 // Make sure user is logged in.
diff --git a/inc/config.php b/inc/config.php
index 8bafd595..bd9287fe 100644
--- a/inc/config.php
+++ b/inc/config.php
@@ -14,9 +14,12 @@ if (!is_dir('db/')) {
14 @mkdir('db/',0705); 14 @mkdir('db/',0705);
15} 15}
16 16
17define ('MODE_DEMO', FALSE);
17define ('ABS_PATH', 'assets/'); 18define ('ABS_PATH', 'assets/');
18define ('CONVERT_LINKS_FOOTNOTES', TRUE); 19define ('CONVERT_LINKS_FOOTNOTES', TRUE);
20define ('REVERT_FORCED_PARAGRAPH_ELEMENTS',FALSE);
19define ('DOWNLOAD_PICTURES', TRUE); 21define ('DOWNLOAD_PICTURES', TRUE);
22define ('SALT', '464v54gLLw928uz4zUBqkRJeiPY68zCX');
20$storage_type = 'sqlite'; # sqlite or file 23$storage_type = 'sqlite'; # sqlite or file
21 24
22include 'functions.php'; 25include 'functions.php';
@@ -32,9 +35,7 @@ require_once 'class.messages.php';
32 35
33Session::init(); 36Session::init();
34 37
35$store = new $storage_type(); 38$store = new $storage_type();
36$msg = new Messages();
37
38# initialisation de RainTPL 39# initialisation de RainTPL
39raintpl::$tpl_dir = './tpl/'; 40raintpl::$tpl_dir = './tpl/';
40raintpl::$cache_dir = './cache/'; 41raintpl::$cache_dir = './cache/';
@@ -42,4 +43,24 @@ raintpl::$base_url = get_poche_url();
42raintpl::configure('path_replace', false); 43raintpl::configure('path_replace', false);
43raintpl::configure('debug', false); 44raintpl::configure('debug', false);
44$tpl = new raintpl(); 45$tpl = new raintpl();
46
47if(!$store->isInstalled())
48{
49 logm('poche still not installed');
50 $tpl->draw('install');
51 if (isset($_GET['install'])) {
52 if (($_POST['password'] == $_POST['password_repeat'])
53 && $_POST['password'] != "" && $_POST['login'] != "") {
54 $store->install($_POST['login'], encode_string($_POST['password'] . $_POST['login']));
55 Session::logout();
56 MyTool::redirect();
57 }
58 }
59 exit();
60}
61
62$_SESSION['login'] = (isset ($_SESSION['login'])) ? $_SESSION['login'] : $store->getLogin();
63$_SESSION['pass'] = (isset ($_SESSION['pass'])) ? $_SESSION['pass'] : $store->getPassword();
64
65$msg = new Messages();
45$tpl->assign('msg', $msg); \ No newline at end of file 66$tpl->assign('msg', $msg); \ No newline at end of file
diff --git a/inc/functions.php b/inc/functions.php
index 205f3968..73e591c5 100644
--- a/inc/functions.php
+++ b/inc/functions.php
@@ -23,6 +23,11 @@ function get_poche_url()
23 return $protocol . "://" . $_SERVER['HTTP_HOST'] . $_SERVER['REQUEST_URI']; 23 return $protocol . "://" . $_SERVER['HTTP_HOST'] . $_SERVER['REQUEST_URI'];
24} 24}
25 25
26function encode_string($string)
27{
28 return sha1($string . SALT);
29}
30
26// function define to retrieve url content 31// function define to retrieve url content
27function get_external_file($url) 32function get_external_file($url)
28{ 33{
@@ -39,6 +44,10 @@ function get_external_file($url)
39 curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); 44 curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
40 curl_setopt($curl, CURLOPT_HEADER, false); 45 curl_setopt($curl, CURLOPT_HEADER, false);
41 46
47 // FOR SSL do not verified certificate
48 curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
49 curl_setopt($curl, CURLOPT_AUTOREFERER, TRUE );
50
42 // FeedBurner requires a proper USER-AGENT... 51 // FeedBurner requires a proper USER-AGENT...
43 curl_setopt($curl, CURL_HTTP_VERSION_1_1, true); 52 curl_setopt($curl, CURL_HTTP_VERSION_1_1, true);
44 curl_setopt($curl, CURLOPT_ENCODING, "gzip, deflate"); 53 curl_setopt($curl, CURLOPT_ENCODING, "gzip, deflate");
@@ -54,7 +63,15 @@ function get_external_file($url)
54 } else { 63 } else {
55 64
56 // create http context and add timeout and user-agent 65 // create http context and add timeout and user-agent
57 $context = stream_context_create(array('http'=>array('timeout' => $timeout,'header'=> "User-Agent: ".$useragent,/*spoot Mozilla Firefox*/'follow_location' => true))); 66 $context = stream_context_create(array(
67 'http'=>array('timeout' => $timeout,
68 'header'=> "User-Agent: ".$useragent, /*spoot Mozilla Firefox*/
69 'follow_location' => true),
70 // FOR SSL do not verified certificate
71 'ssl' => array('verify_peer' => false,
72 'allow_self_signed' => true)
73 )
74 );
58 75
59 // only download page lesser than 4MB 76 // only download page lesser than 4MB
60 $data = @file_get_contents($url, false, $context, -1, 4000000); // We download at most 4 MB from source. 77 $data = @file_get_contents($url, false, $context, -1, 4000000); // We download at most 4 MB from source.
@@ -108,14 +125,26 @@ function prepare_url($url)
108 $i=strpos($url,'#xtor=RSS-'); if ($i!==false) $url=substr($url,0,$i); 125 $i=strpos($url,'#xtor=RSS-'); if ($i!==false) $url=substr($url,0,$i);
109 126
110 $title = $url; 127 $title = $url;
111 if (!preg_match('!^https?://!i', $url)) 128 $html = Encoding::toUTF8(get_external_file($url,15));
129 // If get_external_file if not able to retrieve HTTPS content try the same URL with HTTP protocol
130 if (!preg_match('!^https?://!i', $url) && (!isset($html) || strlen($html) <= 0)) {
112 $url = 'http://' . $url; 131 $url = 'http://' . $url;
132 $html = Encoding::toUTF8(get_external_file($url,15));
133 }
134
135 if (function_exists('tidy_parse_string')) {
136 $tidy = tidy_parse_string($html, array(), 'UTF8');
137 $tidy->cleanRepair();
138 $html = $tidy->value;
139 }
113 140
114 $html = Encoding::toUTF8(get_external_file($url,15));
115 if (isset($html) and strlen($html) > 0) 141 if (isset($html) and strlen($html) > 0)
116 { 142 {
117 $r = new Readability($html, $url); 143 $r = new Readability($html, $url);
144
118 $r->convertLinksToFootnotes = CONVERT_LINKS_FOOTNOTES; 145 $r->convertLinksToFootnotes = CONVERT_LINKS_FOOTNOTES;
146 $r->revertForcedParagraphElements = REVERT_FORCED_PARAGRAPH_ELEMENTS;
147
119 if($r->init()) 148 if($r->init())
120 { 149 {
121 $content = $r->articleContent->innerHTML; 150 $content = $r->articleContent->innerHTML;
@@ -125,8 +154,6 @@ function prepare_url($url)
125 } 154 }
126 } 155 }
127 156
128 $msg->add('e', 'error during url preparation');
129 logm('error during url preparation');
130 return FALSE; 157 return FALSE;
131} 158}
132 159
@@ -263,7 +290,13 @@ function display_view($view, $id = 0, $full_head = 'yes')
263 $tpl->assign('id', $entry['id']); 290 $tpl->assign('id', $entry['id']);
264 $tpl->assign('url', $entry['url']); 291 $tpl->assign('url', $entry['url']);
265 $tpl->assign('title', $entry['title']); 292 $tpl->assign('title', $entry['title']);
266 $tpl->assign('content', $entry['content']); 293 $content = $entry['content'];
294 if (function_exists('tidy_parse_string')) {
295 $tidy = tidy_parse_string($content, array('indent'=>true, 'show-body-only' => true), 'UTF8');
296 $tidy->cleanRepair();
297 $content = $tidy->value;
298 }
299 $tpl->assign('content', $content);
267 $tpl->assign('is_fav', $entry['is_fav']); 300 $tpl->assign('is_fav', $entry['is_fav']);
268 $tpl->assign('is_read', $entry['is_read']); 301 $tpl->assign('is_read', $entry['is_read']);
269 $tpl->assign('load_all_js', 0); 302 $tpl->assign('load_all_js', 0);
@@ -311,35 +344,46 @@ function action_to_do($action, $url, $id = 0)
311 344
312 if (MyTool::isUrl($url)) { 345 if (MyTool::isUrl($url)) {
313 if($parametres_url = prepare_url($url)) { 346 if($parametres_url = prepare_url($url)) {
314 $store->add($url, $parametres_url['title'], $parametres_url['content']); 347 if ($store->add($url, $parametres_url['title'], $parametres_url['content'])) {
315 $last_id = $store->getLastId(); 348 $last_id = $store->getLastId();
316 if (DOWNLOAD_PICTURES) { 349 if (DOWNLOAD_PICTURES) {
317 $content = filtre_picture($parametres_url['content'], $url, $last_id); 350 $content = filtre_picture($parametres_url['content'], $url, $last_id);
351 }
352 $msg->add('s', 'the link has been added successfully');
318 } 353 }
319 $msg->add('s', 'the link has been added successfully'); 354 else {
355 $msg->add('e', 'error during insertion : the link wasn\'t added');
356 }
357 }
358 else {
359 $msg->add('e', 'error during url preparation : the link wasn\'t added');
360 logm('error during url preparation');
320 } 361 }
321 } 362 }
322 else { 363 else {
323 $msg->add('e', 'the link has been added successfully'); 364 $msg->add('e', 'error during url preparation : the link is not valid');
324 logm($url . ' is not a valid url'); 365 logm($url . ' is not a valid url');
325 } 366 }
326 367
327 logm('add link ' . $url); 368 logm('add link ' . $url);
328 break; 369 break;
329 case 'delete': 370 case 'delete':
330 remove_directory(ABS_PATH . $id); 371 if ($store->deleteById($id)) {
331 $store->deleteById($id); 372 remove_directory(ABS_PATH . $id);
332 $msg->add('s', 'the link has been deleted successfully'); 373 $msg->add('s', 'the link has been deleted successfully');
333 logm('delete link #' . $id); 374 logm('delete link #' . $id);
375 }
376 else {
377 $msg->add('e', 'the link wasn\'t deleted');
378 logm('error : can\'t delete link #' . $id);
379 }
334 break; 380 break;
335 case 'toggle_fav' : 381 case 'toggle_fav' :
336 $store->favoriteById($id); 382 $store->favoriteById($id);
337 $msg->add('s', 'the favorite toggle has been done successfully');
338 logm('mark as favorite link #' . $id); 383 logm('mark as favorite link #' . $id);
339 break; 384 break;
340 case 'toggle_archive' : 385 case 'toggle_archive' :
341 $store->archiveById($id); 386 $store->archiveById($id);
342 $msg->add('s', 'the archive toggle has been done successfully');
343 logm('archive link #' . $id); 387 logm('archive link #' . $id);
344 break; 388 break;
345 default: 389 default:
@@ -351,4 +395,4 @@ function logm($message)
351{ 395{
352 $t = strval(date('Y/m/d_H:i:s')).' - '.$_SERVER["REMOTE_ADDR"].' - '.strval($message)."\n"; 396 $t = strval(date('Y/m/d_H:i:s')).' - '.$_SERVER["REMOTE_ADDR"].' - '.strval($message)."\n";
353 file_put_contents('./log.txt',$t,FILE_APPEND); 397 file_put_contents('./log.txt',$t,FILE_APPEND);
354} \ No newline at end of file 398}
diff --git a/inc/store/sqlite.class.php b/inc/store/sqlite.class.php
index d5208a29..21081608 100644
--- a/inc/store/sqlite.class.php
+++ b/inc/store/sqlite.class.php
@@ -17,7 +17,6 @@ class Sqlite extends Store {
17 parent::__construct(); 17 parent::__construct();
18 18
19 $this->handle = new PDO(self::$db_path); 19 $this->handle = new PDO(self::$db_path);
20 $this->handle->exec('CREATE TABLE IF NOT EXISTS "entries" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE , "title" VARCHAR, "url" VARCHAR UNIQUE , "is_read" INTEGER DEFAULT 0, "is_fav" INTEGER DEFAULT 0, "content" BLOB)');
21 $this->handle->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); 20 $this->handle->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
22 } 21 }
23 22
@@ -25,6 +24,63 @@ class Sqlite extends Store {
25 return $this->handle; 24 return $this->handle;
26 } 25 }
27 26
27 public function isInstalled() {
28 $sql = "SELECT name FROM sqlite_sequence WHERE name=?";
29 $query = $this->executeQuery($sql, array('config'));
30 $hasConfig = $query->fetchAll();
31
32 if (count($hasConfig) == 0)
33 return FALSE;
34
35 if (!$this->getLogin() || !$this->getPassword())
36 return FALSE;
37
38 return TRUE;
39 }
40
41 public function install($login, $password) {
42 $this->getHandle()->exec('CREATE TABLE IF NOT EXISTS "config" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE , "name" VARCHAR UNIQUE, "value" BLOB)');
43
44 $this->handle->exec('CREATE TABLE IF NOT EXISTS "entries" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL UNIQUE , "title" VARCHAR, "url" VARCHAR UNIQUE , "is_read" INTEGER DEFAULT 0, "is_fav" INTEGER DEFAULT 0, "content" BLOB)');
45
46 if (!$this->getLogin()) {
47 $sql_login = 'INSERT INTO config ( name, value ) VALUES (?, ?)';
48 $params_login = array('login', $login);
49 $query = $this->executeQuery($sql_login, $params_login);
50 }
51
52 if (!$this->getPassword()) {
53 $sql_pass = 'INSERT INTO config ( name, value ) VALUES (?, ?)';
54 $params_pass = array('password', $password);
55 $query = $this->executeQuery($sql_pass, $params_pass);
56 }
57
58 return TRUE;
59 }
60
61 public function getLogin() {
62 $sql = "SELECT value FROM config WHERE name=?";
63 $query = $this->executeQuery($sql, array('login'));
64 $login = $query->fetchAll();
65
66 return isset($login[0]['value']) ? $login[0]['value'] : FALSE;
67 }
68
69 public function getPassword() {
70 $sql = "SELECT value FROM config WHERE name=?";
71 $query = $this->executeQuery($sql, array('password'));
72 $pass = $query->fetchAll();
73
74 return isset($pass[0]['value']) ? $pass[0]['value'] : FALSE;
75 }
76
77 public function updatePassword($password)
78 {
79 $sql_update = "UPDATE config SET value=? WHERE name='password'";
80 $params_update = array($password);
81 $query = $this->executeQuery($sql_update, $params_update);
82 }
83
28 private function executeQuery($sql, $params) { 84 private function executeQuery($sql, $params) {
29 try 85 try
30 { 86 {
@@ -107,6 +163,7 @@ class Sqlite extends Store {
107 $sql_action = 'INSERT INTO entries ( url, title, content ) VALUES (?, ?, ?)'; 163 $sql_action = 'INSERT INTO entries ( url, title, content ) VALUES (?, ?, ?)';
108 $params_action = array($url, $title, $content); 164 $params_action = array($url, $title, $content);
109 $query = $this->executeQuery($sql_action, $params_action); 165 $query = $this->executeQuery($sql_action, $params_action);
166 return $query;
110 } 167 }
111 168
112 public function deleteById($id) { 169 public function deleteById($id) {
@@ -114,6 +171,7 @@ class Sqlite extends Store {
114 $sql_action = "DELETE FROM entries WHERE id=?"; 171 $sql_action = "DELETE FROM entries WHERE id=?";
115 $params_action = array($id); 172 $params_action = array($id);
116 $query = $this->executeQuery($sql_action, $params_action); 173 $query = $this->executeQuery($sql_action, $params_action);
174 return $query;
117 } 175 }
118 176
119 public function favoriteById($id) { 177 public function favoriteById($id) {
diff --git a/inc/store/store.class.php b/inc/store/store.class.php
index 360ff7c2..dd7d4cfe 100644
--- a/inc/store/store.class.php
+++ b/inc/store/store.class.php
@@ -13,6 +13,14 @@ class Store {
13 13
14 } 14 }
15 15
16 public function getLogin() {
17
18 }
19
20 public function getPassword() {
21
22 }
23
16 public function add() { 24 public function add() {
17 25
18 } 26 }
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 09:57:14 +0000