diff options
Diffstat (limited to 'inc/3rdparty/makefulltextfeed.php')
-rw-r--r-- | inc/3rdparty/makefulltextfeed.php | 1195 |
1 files changed, 1195 insertions, 0 deletions
diff --git a/inc/3rdparty/makefulltextfeed.php b/inc/3rdparty/makefulltextfeed.php new file mode 100644 index 00000000..7104bc73 --- /dev/null +++ b/inc/3rdparty/makefulltextfeed.php | |||
@@ -0,0 +1,1195 @@ | |||
1 | <?php | ||
2 | // Full-Text RSS: Create Full-Text Feeds | ||
3 | // Author: Keyvan Minoukadeh | ||
4 | // Copyright (c) 2013 Keyvan Minoukadeh | ||
5 | // License: AGPLv3 | ||
6 | // Version: 3.1 | ||
7 | // Date: 2013-03-05 | ||
8 | // More info: http://fivefilters.org/content-only/ | ||
9 | // Help: http://help.fivefilters.org | ||
10 | |||
11 | /* | ||
12 | This program is free software: you can redistribute it and/or modify | ||
13 | it under the terms of the GNU Affero General Public License as published by | ||
14 | the Free Software Foundation, either version 3 of the License, or | ||
15 | (at your option) any later version. | ||
16 | |||
17 | This program is distributed in the hope that it will be useful, | ||
18 | but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
19 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
20 | GNU Affero General Public License for more details. | ||
21 | |||
22 | You should have received a copy of the GNU Affero General Public License | ||
23 | along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
24 | */ | ||
25 | |||
26 | // Usage | ||
27 | // ----- | ||
28 | // Request this file passing it your feed in the querystring: makefulltextfeed.php?url=mysite.org | ||
29 | // The following options can be passed in the querystring: | ||
30 | // * URL: url=[feed or website url] (required, should be URL-encoded - in php: urlencode($url)) | ||
31 | // * URL points to HTML (not feed): html=true (optional, by default it's automatically detected) | ||
32 | // * API key: key=[api key] (optional, refer to config.php) | ||
33 | // * Max entries to process: max=[max number of items] (optional) | ||
34 | |||
35 | error_reporting(E_ALL ^ E_NOTICE); | ||
36 | ini_set("display_errors", 1); | ||
37 | @set_time_limit(120); | ||
38 | |||
39 | // Deal with magic quotes | ||
40 | if (get_magic_quotes_gpc()) { | ||
41 | $process = array(&$_GET, &$_POST, &$_REQUEST); | ||
42 | while (list($key, $val) = each($process)) { | ||
43 | foreach ($val as $k => $v) { | ||
44 | unset($process[$key][$k]); | ||
45 | if (is_array($v)) { | ||
46 | $process[$key][stripslashes($k)] = $v; | ||
47 | $process[] = &$process[$key][stripslashes($k)]; | ||
48 | } else { | ||
49 | $process[$key][stripslashes($k)] = stripslashes($v); | ||
50 | } | ||
51 | } | ||
52 | } | ||
53 | unset($process); | ||
54 | } | ||
55 | |||
56 | // set include path | ||
57 | set_include_path(realpath(dirname(__FILE__).'/libraries').PATH_SEPARATOR.get_include_path()); | ||
58 | // Autoloading of classes allows us to include files only when they're | ||
59 | // needed. If we've got a cached copy, for example, only Zend_Cache is loaded. | ||
60 | function autoload($class_name) { | ||
61 | static $dir = null; | ||
62 | if ($dir === null) $dir = dirname(__FILE__).'/libraries/'; | ||
63 | static $mapping = array( | ||
64 | // Include FeedCreator for RSS/Atom creation | ||
65 | 'FeedWriter' => 'feedwriter/FeedWriter.php', | ||
66 | 'FeedItem' => 'feedwriter/FeedItem.php', | ||
67 | // Include ContentExtractor and Readability for identifying and extracting content from URLs | ||
68 | 'ContentExtractor' => 'content-extractor/ContentExtractor.php', | ||
69 | 'SiteConfig' => 'content-extractor/SiteConfig.php', | ||
70 | 'Readability' => 'readability/Readability.php', | ||
71 | // Include Humble HTTP Agent to allow parallel requests and response caching | ||
72 | 'HumbleHttpAgent' => 'humble-http-agent/HumbleHttpAgent.php', | ||
73 | 'SimplePie_HumbleHttpAgent' => 'humble-http-agent/SimplePie_HumbleHttpAgent.php', | ||
74 | 'CookieJar' => 'humble-http-agent/CookieJar.php', | ||
75 | // Include Zend Cache to improve performance (cache results) | ||
76 | 'Zend_Cache' => 'Zend/Cache.php', | ||
77 | // Language detect | ||
78 | 'Text_LanguageDetect' => 'language-detect/LanguageDetect.php', | ||
79 | // HTML5 Lib | ||
80 | 'HTML5_Parser' => 'html5/Parser.php', | ||
81 | // htmLawed - used if XSS filter is enabled (xss_filter) | ||
82 | 'htmLawed' => 'htmLawed/htmLawed.php' | ||
83 | ); | ||
84 | if (isset($mapping[$class_name])) { | ||
85 | debug("** Loading class $class_name ({$mapping[$class_name]})"); | ||
86 | require $dir.$mapping[$class_name]; | ||
87 | return true; | ||
88 | } else { | ||
89 | return false; | ||
90 | } | ||
91 | } | ||
92 | spl_autoload_register('autoload'); | ||
93 | require dirname(__FILE__).'/libraries/simplepie/autoloader.php'; | ||
94 | |||
95 | //////////////////////////////// | ||
96 | // Load config file | ||
97 | //////////////////////////////// | ||
98 | require dirname(__FILE__).'/config.php'; | ||
99 | |||
100 | //////////////////////////////// | ||
101 | // Prevent indexing/following by search engines because: | ||
102 | // 1. The content is already public and presumably indexed (why create duplicates?) | ||
103 | // 2. Not doing so might increase number of requests from search engines, thus increasing server load | ||
104 | // Note: feed readers and services such as Yahoo Pipes will not be affected by this header. | ||
105 | // Note: Using Disallow in a robots.txt file will be more effective (search engines will check | ||
106 | // that before even requesting makefulltextfeed.php). | ||
107 | //////////////////////////////// | ||
108 | header('X-Robots-Tag: noindex, nofollow'); | ||
109 | |||
110 | //////////////////////////////// | ||
111 | // Check if service is enabled | ||
112 | //////////////////////////////// | ||
113 | if (!$options->enabled) { | ||
114 | die('The full-text RSS service is currently disabled'); | ||
115 | } | ||
116 | |||
117 | //////////////////////////////// | ||
118 | // Debug mode? | ||
119 | // See the config file for debug options. | ||
120 | //////////////////////////////// | ||
121 | $debug_mode = false; | ||
122 | if (isset($_GET['debug'])) { | ||
123 | if ($options->debug === true || $options->debug == 'user') { | ||
124 | $debug_mode = true; | ||
125 | } elseif ($options->debug == 'admin') { | ||
126 | session_start(); | ||
127 | $debug_mode = (@$_SESSION['auth'] == 1); | ||
128 | } | ||
129 | if ($debug_mode) { | ||
130 | header('Content-Type: text/plain; charset=utf-8'); | ||
131 | } else { | ||
132 | if ($options->debug == 'admin') { | ||
133 | die('You must be logged in to the <a href="admin/">admin area</a> to see debug output.'); | ||
134 | } else { | ||
135 | die('Debugging is disabled.'); | ||
136 | } | ||
137 | } | ||
138 | } | ||
139 | |||
140 | //////////////////////////////// | ||
141 | // Check for APC | ||
142 | //////////////////////////////// | ||
143 | $options->apc = $options->apc && function_exists('apc_add'); | ||
144 | if ($options->apc) { | ||
145 | debug('APC is enabled and available on server'); | ||
146 | } else { | ||
147 | debug('APC is disabled or not available on server'); | ||
148 | } | ||
149 | |||
150 | //////////////////////////////// | ||
151 | // Check for smart cache | ||
152 | //////////////////////////////// | ||
153 | $options->smart_cache = $options->smart_cache && function_exists('apc_inc'); | ||
154 | |||
155 | //////////////////////////////// | ||
156 | // Check for feed URL | ||
157 | //////////////////////////////// | ||
158 | if (!isset($_GET['url'])) { | ||
159 | die('No URL supplied'); | ||
160 | } | ||
161 | $url = trim($_GET['url']); | ||
162 | if (strtolower(substr($url, 0, 7)) == 'feed://') { | ||
163 | $url = 'http://'.substr($url, 7); | ||
164 | } | ||
165 | if (!preg_match('!^https?://.+!i', $url)) { | ||
166 | $url = 'http://'.$url; | ||
167 | } | ||
168 | |||
169 | $url = filter_var($url, FILTER_SANITIZE_URL); | ||
170 | $test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED); | ||
171 | // deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2) | ||
172 | if ($test === false) { | ||
173 | $test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED); | ||
174 | } | ||
175 | if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) { | ||
176 | // all okay | ||
177 | unset($test); | ||
178 | } else { | ||
179 | die('Invalid URL supplied'); | ||
180 | } | ||
181 | debug("Supplied URL: $url"); | ||
182 | |||
183 | ///////////////////////////////// | ||
184 | // Redirect to hide API key | ||
185 | ///////////////////////////////// | ||
186 | if (isset($_GET['key']) && ($key_index = array_search($_GET['key'], $options->api_keys)) !== false) { | ||
187 | $host = $_SERVER['HTTP_HOST']; | ||
188 | $path = rtrim(dirname($_SERVER['SCRIPT_NAME']), '/\\'); | ||
189 | $_qs_url = (strtolower(substr($url, 0, 7)) == 'http://') ? substr($url, 7) : $url; | ||
190 | $redirect = 'http://'.htmlspecialchars($host.$path).'/makefulltextfeed.php?url='.urlencode($_qs_url); | ||
191 | $redirect .= '&key='.$key_index; | ||
192 | $redirect .= '&hash='.urlencode(sha1($_GET['key'].$url)); | ||
193 | if (isset($_GET['html'])) $redirect .= '&html='.urlencode($_GET['html']); | ||
194 | if (isset($_GET['max'])) $redirect .= '&max='.(int)$_GET['max']; | ||
195 | if (isset($_GET['links'])) $redirect .= '&links='.urlencode($_GET['links']); | ||
196 | if (isset($_GET['exc'])) $redirect .= '&exc='.urlencode($_GET['exc']); | ||
197 | if (isset($_GET['format'])) $redirect .= '&format='.urlencode($_GET['format']); | ||
198 | if (isset($_GET['callback'])) $redirect .= '&callback='.urlencode($_GET['callback']); | ||
199 | if (isset($_GET['l'])) $redirect .= '&l='.urlencode($_GET['l']); | ||
200 | if (isset($_GET['xss'])) $redirect .= '&xss'; | ||
201 | if (isset($_GET['use_extracted_title'])) $redirect .= '&use_extracted_title'; | ||
202 | if (isset($_GET['debug'])) $redirect .= '&debug'; | ||
203 | if ($debug_mode) { | ||
204 | debug('Redirecting to hide access key, follow URL below to continue'); | ||
205 | debug("Location: $redirect"); | ||
206 | } else { | ||
207 | header("Location: $redirect"); | ||
208 | } | ||
209 | exit; | ||
210 | } | ||
211 | |||
212 | /////////////////////////////////////////////// | ||
213 | // Set timezone. | ||
214 | // Prevents warnings, but needs more testing - | ||
215 | // perhaps if timezone is set in php.ini we | ||
216 | // don't need to set it at all... | ||
217 | /////////////////////////////////////////////// | ||
218 | if (!ini_get('date.timezone') || !@date_default_timezone_set(ini_get('date.timezone'))) { | ||
219 | date_default_timezone_set('UTC'); | ||
220 | } | ||
221 | |||
222 | /////////////////////////////////////////////// | ||
223 | // Check if the request is explicitly for an HTML page | ||
224 | /////////////////////////////////////////////// | ||
225 | $html_only = (isset($_GET['html']) && ($_GET['html'] == '1' || $_GET['html'] == 'true')); | ||
226 | |||
227 | /////////////////////////////////////////////// | ||
228 | // Check if valid key supplied | ||
229 | /////////////////////////////////////////////// | ||
230 | $valid_key = false; | ||
231 | if (isset($_GET['key']) && isset($_GET['hash']) && isset($options->api_keys[(int)$_GET['key']])) { | ||
232 | $valid_key = ($_GET['hash'] == sha1($options->api_keys[(int)$_GET['key']].$url)); | ||
233 | } | ||
234 | $key_index = ($valid_key) ? (int)$_GET['key'] : 0; | ||
235 | if (!$valid_key && $options->key_required) { | ||
236 | die('A valid key must be supplied'); | ||
237 | } | ||
238 | if (!$valid_key && isset($_GET['key']) && $_GET['key'] != '') { | ||
239 | die('The entered key is invalid'); | ||
240 | } | ||
241 | |||
242 | if (file_exists('custom_init.php')) require 'custom_init.php'; | ||
243 | |||
244 | /////////////////////////////////////////////// | ||
245 | // Check URL against list of blacklisted URLs | ||
246 | /////////////////////////////////////////////// | ||
247 | if (!url_allowed($url)) die('URL blocked'); | ||
248 | |||
249 | /////////////////////////////////////////////// | ||
250 | // Max entries | ||
251 | // see config.php to find these values | ||
252 | /////////////////////////////////////////////// | ||
253 | if (isset($_GET['max'])) { | ||
254 | $max = (int)$_GET['max']; | ||
255 | if ($valid_key) { | ||
256 | $max = min($max, $options->max_entries_with_key); | ||
257 | } else { | ||
258 | $max = min($max, $options->max_entries); | ||
259 | } | ||
260 | } else { | ||
261 | if ($valid_key) { | ||
262 | $max = $options->default_entries_with_key; | ||
263 | } else { | ||
264 | $max = $options->default_entries; | ||
265 | } | ||
266 | } | ||
267 | |||
268 | /////////////////////////////////////////////// | ||
269 | // Link handling | ||
270 | /////////////////////////////////////////////// | ||
271 | if (isset($_GET['links']) && in_array($_GET['links'], array('preserve', 'footnotes', 'remove'))) { | ||
272 | $links = $_GET['links']; | ||
273 | } else { | ||
274 | $links = 'preserve'; | ||
275 | } | ||
276 | |||
277 | /////////////////////////////////////////////// | ||
278 | // Favour item titles in feed? | ||
279 | /////////////////////////////////////////////// | ||
280 | $favour_feed_titles = true; | ||
281 | if ($options->favour_feed_titles == 'user') { | ||
282 | $favour_feed_titles = !isset($_GET['use_extracted_title']); | ||
283 | } else { | ||
284 | $favour_feed_titles = $options->favour_feed_titles; | ||
285 | } | ||
286 | |||
287 | /////////////////////////////////////////////// | ||
288 | // Exclude items if extraction fails | ||
289 | /////////////////////////////////////////////// | ||
290 | if ($options->exclude_items_on_fail === 'user') { | ||
291 | $exclude_on_fail = (isset($_GET['exc']) && ($_GET['exc'] == '1')); | ||
292 | } else { | ||
293 | $exclude_on_fail = $options->exclude_items_on_fail; | ||
294 | } | ||
295 | |||
296 | /////////////////////////////////////////////// | ||
297 | // Detect language | ||
298 | /////////////////////////////////////////////// | ||
299 | if ($options->detect_language === 'user') { | ||
300 | if (isset($_GET['l'])) { | ||
301 | $detect_language = (int)$_GET['l']; | ||
302 | } else { | ||
303 | $detect_language = 1; | ||
304 | } | ||
305 | } else { | ||
306 | $detect_language = $options->detect_language; | ||
307 | } | ||
308 | |||
309 | if ($detect_language >= 2) { | ||
310 | $language_codes = array('albanian' => 'sq','arabic' => 'ar','azeri' => 'az','bengali' => 'bn','bulgarian' => 'bg', | ||
311 | 'cebuano' => 'ceb', // ISO 639-2 | ||
312 | 'croatian' => 'hr','czech' => 'cs','danish' => 'da','dutch' => 'nl','english' => 'en','estonian' => 'et','farsi' => 'fa','finnish' => 'fi','french' => 'fr','german' => 'de','hausa' => 'ha', | ||
313 | 'hawaiian' => 'haw', // ISO 639-2 | ||
314 | 'hindi' => 'hi','hungarian' => 'hu','icelandic' => 'is','indonesian' => 'id','italian' => 'it','kazakh' => 'kk','kyrgyz' => 'ky','latin' => 'la','latvian' => 'lv','lithuanian' => 'lt','macedonian' => 'mk','mongolian' => 'mn','nepali' => 'ne','norwegian' => 'no','pashto' => 'ps', | ||
315 | 'pidgin' => 'cpe', // ISO 639-2 | ||
316 | 'polish' => 'pl','portuguese' => 'pt','romanian' => 'ro','russian' => 'ru','serbian' => 'sr','slovak' => 'sk','slovene' => 'sl','somali' => 'so','spanish' => 'es','swahili' => 'sw','swedish' => 'sv','tagalog' => 'tl','turkish' => 'tr','ukrainian' => 'uk','urdu' => 'ur','uzbek' => 'uz','vietnamese' => 'vi','welsh' => 'cy'); | ||
317 | } | ||
318 | $use_cld = extension_loaded('cld') && (version_compare(PHP_VERSION, '5.3.0') >= 0); | ||
319 | |||
320 | ///////////////////////////////////// | ||
321 | // Check for valid format | ||
322 | // (stick to RSS (or RSS as JSON) for the time being) | ||
323 | ///////////////////////////////////// | ||
324 | if (isset($_GET['format']) && $_GET['format'] == 'json') { | ||
325 | $format = 'json'; | ||
326 | } else { | ||
327 | $format = 'rss'; | ||
328 | } | ||
329 | |||
330 | ///////////////////////////////////// | ||
331 | // Should we do XSS filtering? | ||
332 | ///////////////////////////////////// | ||
333 | if ($options->xss_filter === 'user') { | ||
334 | $xss_filter = isset($_GET['xss']); | ||
335 | } else { | ||
336 | $xss_filter = $options->xss_filter; | ||
337 | } | ||
338 | if (!$xss_filter && isset($_GET['xss'])) { | ||
339 | die('XSS filtering is disabled in config'); | ||
340 | } | ||
341 | |||
342 | ///////////////////////////////////// | ||
343 | // Check for JSONP | ||
344 | // Regex from https://gist.github.com/1217080 | ||
345 | ///////////////////////////////////// | ||
346 | $callback = null; | ||
347 | if ($format =='json' && isset($_GET['callback'])) { | ||
348 | $callback = trim($_GET['callback']); | ||
349 | foreach (explode('.', $callback) as $_identifier) { | ||
350 | if (!preg_match('/^[a-zA-Z_$][0-9a-zA-Z_$]*(?:\[(?:".+"|\'.+\'|\d+)\])*?$/', $_identifier)) { | ||
351 | die('Invalid JSONP callback'); | ||
352 | } | ||
353 | } | ||
354 | debug("JSONP callback: $callback"); | ||
355 | } | ||
356 | |||
357 | ////////////////////////////////// | ||
358 | // Enable Cross-Origin Resource Sharing (CORS) | ||
359 | ////////////////////////////////// | ||
360 | if ($options->cors) header('Access-Control-Allow-Origin: *'); | ||
361 | |||
362 | ////////////////////////////////// | ||
363 | // Check for cached copy | ||
364 | ////////////////////////////////// | ||
365 | if ($options->caching) { | ||
366 | debug('Caching is enabled...'); | ||
367 | $cache_id = md5($max.$url.$valid_key.$links.$favour_feed_titles.$xss_filter.$exclude_on_fail.$format.$detect_language.(int)isset($_GET['pubsub'])); | ||
368 | $check_cache = true; | ||
369 | if ($options->apc && $options->smart_cache) { | ||
370 | apc_add("cache.$cache_id", 0, 10*60); | ||
371 | $apc_cache_hits = (int)apc_fetch("cache.$cache_id"); | ||
372 | $check_cache = ($apc_cache_hits >= 2); | ||
373 | apc_inc("cache.$cache_id"); | ||
374 | if ($check_cache) { | ||
375 | debug('Cache key found in APC, we\'ll try to load cache file from disk'); | ||
376 | } else { | ||
377 | debug('Cache key not found in APC'); | ||
378 | } | ||
379 | } | ||
380 | if ($check_cache) { | ||
381 | $cache = get_cache(); | ||
382 | if ($data = $cache->load($cache_id)) { | ||
383 | if ($debug_mode) { | ||
384 | debug('Loaded cached copy'); | ||
385 | exit; | ||
386 | } | ||
387 | if ($format == 'json') { | ||
388 | if ($callback === null) { | ||
389 | header('Content-type: application/json; charset=UTF-8'); | ||
390 | } else { | ||
391 | header('Content-type: application/javascript; charset=UTF-8'); | ||
392 | } | ||
393 | } else { | ||
394 | header('Content-type: text/xml; charset=UTF-8'); | ||
395 | header('X-content-type-options: nosniff'); | ||
396 | } | ||
397 | if (headers_sent()) die('Some data has already been output, can\'t send RSS file'); | ||
398 | if ($callback) { | ||
399 | echo "$callback($data);"; | ||
400 | } else { | ||
401 | echo $data; | ||
402 | } | ||
403 | exit; | ||
404 | } | ||
405 | } | ||
406 | } | ||
407 | |||
408 | ////////////////////////////////// | ||
409 | // Set Expires header | ||
410 | ////////////////////////////////// | ||
411 | if (!$debug_mode) { | ||
412 | header('Expires: ' . gmdate('D, d M Y H:i:s', time()+(60*10)) . ' GMT'); | ||
413 | } | ||
414 | |||
415 | ////////////////////////////////// | ||
416 | // Set up HTTP agent | ||
417 | ////////////////////////////////// | ||
418 | $http = new HumbleHttpAgent(); | ||
419 | $http->debug = $debug_mode; | ||
420 | $http->userAgentMap = $options->user_agents; | ||
421 | $http->headerOnlyTypes = array_keys($options->content_type_exc); | ||
422 | $http->rewriteUrls = $options->rewrite_url; | ||
423 | |||
424 | ////////////////////////////////// | ||
425 | // Set up Content Extractor | ||
426 | ////////////////////////////////// | ||
427 | $extractor = new ContentExtractor(dirname(__FILE__).'/site_config/custom', dirname(__FILE__).'/site_config/standard'); | ||
428 | $extractor->debug = $debug_mode; | ||
429 | SiteConfig::$debug = $debug_mode; | ||
430 | SiteConfig::use_apc($options->apc); | ||
431 | $extractor->fingerprints = $options->fingerprints; | ||
432 | $extractor->allowedParsers = $options->allowed_parsers; | ||
433 | |||
434 | //////////////////////////////// | ||
435 | // Get RSS/Atom feed | ||
436 | //////////////////////////////// | ||
437 | if (!$html_only) { | ||
438 | debug('--------'); | ||
439 | debug("Attempting to process URL as feed"); | ||
440 | // Send user agent header showing PHP (prevents a HTML response from feedburner) | ||
441 | $http->userAgentDefault = HumbleHttpAgent::UA_PHP; | ||
442 | // configure SimplePie HTTP extension class to use our HumbleHttpAgent instance | ||
443 | SimplePie_HumbleHttpAgent::set_agent($http); | ||
444 | $feed = new SimplePie(); | ||
445 | // some feeds use the text/html content type - force_feed tells SimplePie to process anyway | ||
446 | $feed->force_feed(true); | ||
447 | $feed->set_file_class('SimplePie_HumbleHttpAgent'); | ||
448 | //$feed->set_feed_url($url); // colons appearing in the URL's path get encoded | ||
449 | $feed->feed_url = $url; | ||
450 | $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE); | ||
451 | $feed->set_timeout(20); | ||
452 | $feed->enable_cache(false); | ||
453 | $feed->set_stupidly_fast(true); | ||
454 | $feed->enable_order_by_date(false); // we don't want to do anything to the feed | ||
455 | $feed->set_url_replacements(array()); | ||
456 | // initialise the feed | ||
457 | // the @ suppresses notices which on some servers causes a 500 internal server error | ||
458 | $result = @$feed->init(); | ||
459 | //$feed->handle_content_type(); | ||
460 | //$feed->get_title(); | ||
461 | if ($result && (!is_array($feed->data) || count($feed->data) == 0)) { | ||
462 | die('Sorry, no feed items found'); | ||
463 | } | ||
464 | // from now on, we'll identify ourselves as a browser | ||
465 | $http->userAgentDefault = HumbleHttpAgent::UA_BROWSER; | ||
466 | } | ||
467 | |||
468 | //////////////////////////////////////////////////////////////////////////////// | ||
469 | // Our given URL is not a feed, so let's create our own feed with a single item: | ||
470 | // the given URL. This basically treats all non-feed URLs as if they were | ||
471 | // single-item feeds. | ||
472 | //////////////////////////////////////////////////////////////////////////////// | ||
473 | $isDummyFeed = false; | ||
474 | if ($html_only || !$result) { | ||
475 | debug('--------'); | ||
476 | debug("Constructing a single-item feed from URL"); | ||
477 | $isDummyFeed = true; | ||
478 | unset($feed, $result); | ||
479 | // create single item dummy feed object | ||
480 | class DummySingleItemFeed { | ||
481 | public $item; | ||
482 | function __construct($url) { $this->item = new DummySingleItem($url); } | ||
483 | public function get_title() { return ''; } | ||
484 | public function get_description() { return 'Content extracted from '.$this->item->url; } | ||
485 | public function get_link() { return $this->item->url; } | ||
486 | public function get_language() { return false; } | ||
487 | public function get_image_url() { return false; } | ||
488 | public function get_items($start=0, $max=1) { return array(0=>$this->item); } | ||
489 | } | ||
490 | class DummySingleItem { | ||
491 | public $url; | ||
492 | function __construct($url) { $this->url = $url; } | ||
493 | public function get_permalink() { return $this->url; } | ||
494 | public function get_title() { return null; } | ||
495 | public function get_date($format='') { return false; } | ||
496 | public function get_author($key=0) { return null; } | ||
497 | public function get_authors() { return null; } | ||
498 | public function get_description() { return ''; } | ||
499 | public function get_enclosure($key=0, $prefer=null) { return null; } | ||
500 | public function get_enclosures() { return null; } | ||
501 | public function get_categories() { return null; } | ||
502 | } | ||
503 | $feed = new DummySingleItemFeed($url); | ||
504 | } | ||
505 | |||
506 | //////////////////////////////////////////// | ||
507 | // Create full-text feed | ||
508 | //////////////////////////////////////////// | ||
509 | $output = new FeedWriter(); | ||
510 | $output->setTitle(strip_tags($feed->get_title())); | ||
511 | $output->setDescription(strip_tags($feed->get_description())); | ||
512 | $output->setXsl('css/feed.xsl'); // Chrome uses this, most browsers ignore it | ||
513 | if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment | ||
514 | $output->addHub('http://fivefilters.superfeedr.com/'); | ||
515 | $output->addHub('http://pubsubhubbub.appspot.com/'); | ||
516 | $output->setSelf('http://'.$_SERVER['HTTP_HOST'].$_SERVER['REQUEST_URI']); | ||
517 | } | ||
518 | $output->setLink($feed->get_link()); // Google Reader uses this for pulling in favicons | ||
519 | if ($img_url = $feed->get_image_url()) { | ||
520 | $output->setImage($feed->get_title(), $feed->get_link(), $img_url); | ||
521 | } | ||
522 | |||
523 | //////////////////////////////////////////// | ||
524 | // Loop through feed items | ||
525 | //////////////////////////////////////////// | ||
526 | $items = $feed->get_items(0, $max); | ||
527 | // Request all feed items in parallel (if supported) | ||
528 | $urls_sanitized = array(); | ||
529 | $urls = array(); | ||
530 | foreach ($items as $key => $item) { | ||
531 | $permalink = htmlspecialchars_decode($item->get_permalink()); | ||
532 | // Colons in URL path segments get encoded by SimplePie, yet some sites expect them unencoded | ||
533 | $permalink = str_replace('%3A', ':', $permalink); | ||
534 | // validateUrl() strips non-ascii characters | ||
535 | // simplepie already sanitizes URLs so let's not do it again here. | ||
536 | //$permalink = $http->validateUrl($permalink); | ||
537 | if ($permalink) { | ||
538 | $urls_sanitized[] = $permalink; | ||
539 | } | ||
540 | $urls[$key] = $permalink; | ||
541 | } | ||
542 | debug('--------'); | ||
543 | debug('Fetching feed items'); | ||
544 | $http->fetchAll($urls_sanitized); | ||
545 | //$http->cacheAll(); | ||
546 | |||
547 | // count number of items added to full feed | ||
548 | $item_count = 0; | ||
549 | |||
550 | foreach ($items as $key => $item) { | ||
551 | debug('--------'); | ||
552 | debug('Processing feed item '.($item_count+1)); | ||
553 | $do_content_extraction = true; | ||
554 | $extract_result = false; | ||
555 | $text_sample = null; | ||
556 | $permalink = $urls[$key]; | ||
557 | debug("Item URL: $permalink"); | ||
558 | $extracted_title = ''; | ||
559 | $feed_item_title = $item->get_title(); | ||
560 | if ($feed_item_title !== null) { | ||
561 | $feed_item_title = strip_tags(htmlspecialchars_decode($feed_item_title)); | ||
562 | } | ||
563 | $newitem = $output->createNewItem(); | ||
564 | $newitem->setTitle($feed_item_title); | ||
565 | if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment | ||
566 | if ($permalink !== false) { | ||
567 | $newitem->setLink('http://fivefilters.org/content-only/redirect.php?url='.urlencode($permalink)); | ||
568 | } else { | ||
569 | $newitem->setLink('http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink())); | ||
570 | } | ||
571 | } else { | ||
572 | if ($permalink !== false) { | ||
573 | $newitem->setLink($permalink); | ||
574 | } else { | ||
575 | $newitem->setLink($item->get_permalink()); | ||
576 | } | ||
577 | } | ||
578 | //if ($permalink && ($response = $http->get($permalink, true)) && $response['status_code'] < 300) { | ||
579 | // Allowing error codes - some sites return correct content with error status | ||
580 | // e.g. prospectmagazine.co.uk returns 403 | ||
581 | if ($permalink && ($response = $http->get($permalink, true)) && ($response['status_code'] < 300 || $response['status_code'] > 400)) { | ||
582 | $effective_url = $response['effective_url']; | ||
583 | if (!url_allowed($effective_url)) continue; | ||
584 | // check if action defined for returned Content-Type | ||
585 | $mime_info = get_mime_action_info($response['headers']); | ||
586 | if (isset($mime_info['action'])) { | ||
587 | if ($mime_info['action'] == 'exclude') { | ||
588 | continue; // skip this feed item entry | ||
589 | } elseif ($mime_info['action'] == 'link') { | ||
590 | if ($mime_info['type'] == 'image') { | ||
591 | $html = "<a href=\"$effective_url\"><img src=\"$effective_url\" alt=\"{$mime_info['name']}\" /></a>"; | ||
592 | } else { | ||
593 | $html = "<a href=\"$effective_url\">Download {$mime_info['name']}</a>"; | ||
594 | } | ||
595 | $extracted_title = $mime_info['name']; | ||
596 | $do_content_extraction = false; | ||
597 | } | ||
598 | } | ||
599 | if ($do_content_extraction) { | ||
600 | $html = $response['body']; | ||
601 | // remove strange things | ||
602 | $html = str_replace('</[>', '', $html); | ||
603 | $html = convert_to_utf8($html, $response['headers']); | ||
604 | // check site config for single page URL - fetch it if found | ||
605 | $is_single_page = false; | ||
606 | if ($single_page_response = getSinglePage($item, $html, $effective_url)) { | ||
607 | $is_single_page = true; | ||
608 | $html = $single_page_response['body']; | ||
609 | // remove strange things | ||
610 | $html = str_replace('</[>', '', $html); | ||
611 | $html = convert_to_utf8($html, $single_page_response['headers']); | ||
612 | $effective_url = $single_page_response['effective_url']; | ||
613 | debug("Retrieved single-page view from $effective_url"); | ||
614 | unset($single_page_response); | ||
615 | } | ||
616 | debug('--------'); | ||
617 | debug('Attempting to extract content'); | ||
618 | $extract_result = $extractor->process($html, $effective_url); | ||
619 | $readability = $extractor->readability; | ||
620 | $content_block = ($extract_result) ? $extractor->getContent() : null; | ||
621 | $extracted_title = ($extract_result) ? $extractor->getTitle() : ''; | ||
622 | // Deal with multi-page articles | ||
623 | //die('Next: '.$extractor->getNextPageUrl()); | ||
624 | $is_multi_page = (!$is_single_page && $extract_result && $extractor->getNextPageUrl()); | ||
625 | if ($options->multipage && $is_multi_page) { | ||
626 | debug('--------'); | ||
627 | debug('Attempting to process multi-page article'); | ||
628 | $multi_page_urls = array(); | ||
629 | $multi_page_content = array(); | ||
630 | while ($next_page_url = $extractor->getNextPageUrl()) { | ||
631 | debug('--------'); | ||
632 | debug('Processing next page: '.$next_page_url); | ||
633 | // If we've got URL, resolve against $url | ||
634 | if ($next_page_url = makeAbsoluteStr($effective_url, $next_page_url)) { | ||
635 | // check it's not what we have already! | ||
636 | if (!in_array($next_page_url, $multi_page_urls)) { | ||
637 | // it's not, so let's attempt to fetch it | ||
638 | $multi_page_urls[] = $next_page_url; | ||
639 | $_prev_ref = $http->referer; | ||
640 | if (($response = $http->get($next_page_url, true)) && $response['status_code'] < 300) { | ||
641 | // make sure mime type is not something with a different action associated | ||
642 | $page_mime_info = get_mime_action_info($response['headers']); | ||
643 | if (!isset($page_mime_info['action'])) { | ||
644 | $html = $response['body']; | ||
645 | // remove strange things | ||
646 | $html = str_replace('</[>', '', $html); | ||
647 | $html = convert_to_utf8($html, $response['headers']); | ||
648 | if ($extractor->process($html, $next_page_url)) { | ||
649 | $multi_page_content[] = $extractor->getContent(); | ||
650 | continue; | ||
651 | } else { debug('Failed to extract content'); } | ||
652 | } else { debug('MIME type requires different action'); } | ||
653 | } else { debug('Failed to fetch URL'); } | ||
654 | } else { debug('URL already processed'); } | ||
655 | } else { debug('Failed to resolve against '.$effective_url); } | ||
656 | // failed to process next_page_url, so cancel further requests | ||
657 | $multi_page_content = array(); | ||
658 | break; | ||
659 | } | ||
660 | // did we successfully deal with this multi-page article? | ||
661 | if (empty($multi_page_content)) { | ||
662 | debug('Failed to extract all parts of multi-page article, so not going to include them'); | ||
663 | $multi_page_content[] = $readability->dom->createElement('p')->innerHTML = '<em>This article appears to continue on subsequent pages which we could not extract</em>'; | ||
664 | } | ||
665 | foreach ($multi_page_content as $_page) { | ||
666 | $_page = $content_block->ownerDocument->importNode($_page, true); | ||
667 | $content_block->appendChild($_page); | ||
668 | } | ||
669 | unset($multi_page_urls, $multi_page_content, $page_mime_info, $next_page_url); | ||
670 | } | ||
671 | } | ||
672 | // use extracted title for both feed and item title if we're using single-item dummy feed | ||
673 | if ($isDummyFeed) { | ||
674 | $output->setTitle($extracted_title); | ||
675 | $newitem->setTitle($extracted_title); | ||
676 | } else { | ||
677 | // use extracted title instead of feed item title? | ||
678 | if (!$favour_feed_titles && $extracted_title != '') { | ||
679 | debug('Using extracted title in generated feed'); | ||
680 | $newitem->setTitle($extracted_title); | ||
681 | } | ||
682 | } | ||
683 | } | ||
684 | if ($do_content_extraction) { | ||
685 | // if we failed to extract content... | ||
686 | if (!$extract_result) { | ||
687 | if ($exclude_on_fail) { | ||
688 | debug('Failed to extract, so skipping (due to exclude on fail parameter)'); | ||
689 | continue; // skip this and move to next item | ||
690 | } | ||
691 | //TODO: get text sample for language detection | ||
692 | $html = $options->error_message; | ||
693 | // keep the original item description | ||
694 | $html .= $item->get_description(); | ||
695 | } else { | ||
696 | $readability->clean($content_block, 'select'); | ||
697 | if ($options->rewrite_relative_urls) makeAbsolute($effective_url, $content_block); | ||
698 | // footnotes | ||
699 | if (($links == 'footnotes') && (strpos($effective_url, 'wikipedia.org') === false)) { | ||
700 | $readability->addFootnotes($content_block); | ||
701 | } | ||
702 | // remove nesting: <div><div><div><p>test</p></div></div></div> = <p>test</p> | ||
703 | while ($content_block->childNodes->length == 1 && $content_block->firstChild->nodeType === XML_ELEMENT_NODE) { | ||
704 | // only follow these tag names | ||
705 | if (!in_array(strtolower($content_block->tagName), array('div', 'article', 'section', 'header', 'footer'))) break; | ||
706 | //$html = $content_block->firstChild->innerHTML; // FTR 2.9.5 | ||
707 | $content_block = $content_block->firstChild; | ||
708 | } | ||
709 | // convert content block to HTML string | ||
710 | // Need to preserve things like body: //img[@id='feature'] | ||
711 | if (in_array(strtolower($content_block->tagName), array('div', 'article', 'section', 'header', 'footer'))) { | ||
712 | $html = $content_block->innerHTML; | ||
713 | } else { | ||
714 | $html = $content_block->ownerDocument->saveXML($content_block); // essentially outerHTML | ||
715 | } | ||
716 | unset($content_block); | ||
717 | // post-processing cleanup | ||
718 | $html = preg_replace('!<p>[\s\h\v]*</p>!u', '', $html); | ||
719 | if ($links == 'remove') { | ||
720 | $html = preg_replace('!</?a[^>]*>!', '', $html); | ||
721 | } | ||
722 | // get text sample for language detection | ||
723 | $text_sample = strip_tags(substr($html, 0, 500)); | ||
724 | $html = make_substitutions($options->message_to_prepend).$html; | ||
725 | $html .= make_substitutions($options->message_to_append); | ||
726 | } | ||
727 | } | ||
728 | |||
729 | if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment | ||
730 | $newitem->addElement('guid', 'http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()), array('isPermaLink'=>'false')); | ||
731 | } else { | ||
732 | $newitem->addElement('guid', $item->get_permalink(), array('isPermaLink'=>'true')); | ||
733 | } | ||
734 | // filter xss? | ||
735 | if ($xss_filter) { | ||
736 | debug('Filtering HTML to remove XSS'); | ||
737 | $html = htmLawed::hl($html, array('safe'=>1, 'deny_attribute'=>'style', 'comment'=>1, 'cdata'=>1)); | ||
738 | } | ||
739 | $newitem->setDescription($html); | ||
740 | |||
741 | // set date | ||
742 | if ((int)$item->get_date('U') > 0) { | ||
743 | $newitem->setDate((int)$item->get_date('U')); | ||
744 | } elseif ($extractor->getDate()) { | ||
745 | $newitem->setDate($extractor->getDate()); | ||
746 | } | ||
747 | |||
748 | // add authors | ||
749 | if ($authors = $item->get_authors()) { | ||
750 | foreach ($authors as $author) { | ||
751 | // for some feeds, SimplePie stores author's name as email, e.g. http://feeds.feedburner.com/nymag/intel | ||
752 | if ($author->get_name() !== null) { | ||
753 | $newitem->addElement('dc:creator', $author->get_name()); | ||
754 | } elseif ($author->get_email() !== null) { | ||
755 | $newitem->addElement('dc:creator', $author->get_email()); | ||
756 | } | ||
757 | } | ||
758 | } elseif ($authors = $extractor->getAuthors()) { | ||
759 | //TODO: make sure the list size is reasonable | ||
760 | foreach ($authors as $author) { | ||
761 | // TODO: xpath often selects authors from other articles linked from the page. | ||
762 | // for now choose first item | ||
763 | $newitem->addElement('dc:creator', $author); | ||
764 | break; | ||
765 | } | ||
766 | } | ||
767 | |||
768 | // add language | ||
769 | if ($detect_language) { | ||
770 | $language = $extractor->getLanguage(); | ||
771 | if (!$language) $language = $feed->get_language(); | ||
772 | if (($detect_language == 3 || (!$language && $detect_language == 2)) && $text_sample) { | ||
773 | try { | ||
774 | if ($use_cld) { | ||
775 | // Use PHP-CLD extension | ||
776 | $php_cld = 'CLD\detect'; // in quotes to prevent PHP 5.2 parse error | ||
777 | $res = $php_cld($text_sample); | ||
778 | if (is_array($res) && count($res) > 0) { | ||
779 | $language = $res[0]['code']; | ||
780 | } | ||
781 | } else { | ||
782 | //die('what'); | ||
783 | // Use PEAR's Text_LanguageDetect | ||
784 | if (!isset($l)) { | ||
785 | $l = new Text_LanguageDetect('libraries/language-detect/lang.dat', 'libraries/language-detect/unicode_blocks.dat'); | ||
786 | } | ||
787 | $l_result = $l->detect($text_sample, 1); | ||
788 | if (count($l_result) > 0) { | ||
789 | $language = $language_codes[key($l_result)]; | ||
790 | } | ||
791 | } | ||
792 | } catch (Exception $e) { | ||
793 | //die('error: '.$e); | ||
794 | // do nothing | ||
795 | } | ||
796 | } | ||
797 | if ($language && (strlen($language) < 7)) { | ||
798 | $newitem->addElement('dc:language', $language); | ||
799 | } | ||
800 | } | ||
801 | |||
802 | // add MIME type (if it appeared in our exclusions lists) | ||
803 | if (isset($mime_info['mime'])) $newitem->addElement('dc:format', $mime_info['mime']); | ||
804 | // add effective URL (URL after redirects) | ||
805 | if (isset($effective_url)) { | ||
806 | //TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g. | ||
807 | //http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir-�-25th-March-2012-Special-Program-from-Liari-(Karachi) | ||
808 | //temporary measure: use utf8_encode() | ||
809 | $newitem->addElement('dc:identifier', remove_url_cruft(utf8_encode($effective_url))); | ||
810 | } else { | ||
811 | $newitem->addElement('dc:identifier', remove_url_cruft($item->get_permalink())); | ||
812 | } | ||
813 | |||
814 | // add categories | ||
815 | if ($categories = $item->get_categories()) { | ||
816 | foreach ($categories as $category) { | ||
817 | if ($category->get_label() !== null) { | ||
818 | $newitem->addElement('category', $category->get_label()); | ||
819 | } | ||
820 | } | ||
821 | } | ||
822 | |||
823 | // check for enclosures | ||
824 | if ($options->keep_enclosures) { | ||
825 | if ($enclosures = $item->get_enclosures()) { | ||
826 | foreach ($enclosures as $enclosure) { | ||
827 | // thumbnails | ||
828 | foreach ((array)$enclosure->get_thumbnails() as $thumbnail) { | ||
829 | $newitem->addElement('media:thumbnail', '', array('url'=>$thumbnail)); | ||
830 | } | ||
831 | if (!$enclosure->get_link()) continue; | ||
832 | $enc = array(); | ||
833 | // Media RSS spec ($enc): http://search.yahoo.com/mrss | ||
834 | // SimplePie methods ($enclosure): http://simplepie.org/wiki/reference/start#methods4 | ||
835 | $enc['url'] = $enclosure->get_link(); | ||
836 | if ($enclosure->get_length()) $enc['fileSize'] = $enclosure->get_length(); | ||
837 | if ($enclosure->get_type()) $enc['type'] = $enclosure->get_type(); | ||
838 | if ($enclosure->get_medium()) $enc['medium'] = $enclosure->get_medium(); | ||
839 | if ($enclosure->get_expression()) $enc['expression'] = $enclosure->get_expression(); | ||
840 | if ($enclosure->get_bitrate()) $enc['bitrate'] = $enclosure->get_bitrate(); | ||
841 | if ($enclosure->get_framerate()) $enc['framerate'] = $enclosure->get_framerate(); | ||
842 | if ($enclosure->get_sampling_rate()) $enc['samplingrate'] = $enclosure->get_sampling_rate(); | ||
843 | if ($enclosure->get_channels()) $enc['channels'] = $enclosure->get_channels(); | ||
844 | if ($enclosure->get_duration()) $enc['duration'] = $enclosure->get_duration(); | ||
845 | if ($enclosure->get_height()) $enc['height'] = $enclosure->get_height(); | ||
846 | if ($enclosure->get_width()) $enc['width'] = $enclosure->get_width(); | ||
847 | if ($enclosure->get_language()) $enc['lang'] = $enclosure->get_language(); | ||
848 | $newitem->addElement('media:content', '', $enc); | ||
849 | } | ||
850 | } | ||
851 | } | ||
852 | /* } */ | ||
853 | $output->addItem($newitem); | ||
854 | unset($html); | ||
855 | $item_count++; | ||
856 | } | ||
857 | |||
858 | // output feed | ||
859 | debug('Done!'); | ||
860 | /* | ||
861 | if ($debug_mode) { | ||
862 | $_apc_data = apc_cache_info('user'); | ||
863 | var_dump($_apc_data); exit; | ||
864 | } | ||
865 | */ | ||
866 | if (!$debug_mode) { | ||
867 | if ($callback) echo "$callback("; // if $callback is set, $format also == 'json' | ||
868 | if ($format == 'json') $output->setFormat(($callback === null) ? JSON : JSONP); | ||
869 | $add_to_cache = $options->caching; | ||
870 | // is smart cache mode enabled? | ||
871 | if ($add_to_cache && $options->apc && $options->smart_cache) { | ||
872 | // yes, so only cache if this is the second request for this URL | ||
873 | $add_to_cache = ($apc_cache_hits >= 2); | ||
874 | // purge cache | ||
875 | if ($options->cache_cleanup > 0) { | ||
876 | if (rand(1, $options->cache_cleanup) == 1) { | ||
877 | // apc purge code adapted from from http://www.thimbleopensource.com/tutorials-snippets/php-apc-expunge-script | ||
878 | $_apc_data = apc_cache_info('user'); | ||
879 | foreach ($_apc_data['cache_list'] as $_apc_item) { | ||
880 | if ($_apc_item['ttl'] > 0 && ($_apc_item['ttl'] + $_apc_item['creation_time'] < time())) { | ||
881 | apc_delete($_apc_item['info']); | ||
882 | } | ||
883 | } | ||
884 | } | ||
885 | } | ||
886 | } | ||
887 | if ($add_to_cache) { | ||
888 | ob_start(); | ||
889 | $output->genarateFeed(); | ||
890 | $output = ob_get_contents(); | ||
891 | ob_end_clean(); | ||
892 | if ($html_only && $item_count == 0) { | ||
893 | // do not cache - in case of temporary server glitch at source URL | ||
894 | } else { | ||
895 | $cache = get_cache(); | ||
896 | if ($add_to_cache) $cache->save($output, $cache_id); | ||
897 | } | ||
898 | echo $output; | ||
899 | } else { | ||
900 | $output->genarateFeed(); | ||
901 | } | ||
902 | if ($callback) echo ');'; | ||
903 | } | ||
904 | |||
905 | /////////////////////////////// | ||
906 | // HELPER FUNCTIONS | ||
907 | /////////////////////////////// | ||
908 | |||
909 | function url_allowed($url) { | ||
910 | global $options; | ||
911 | if (!empty($options->allowed_urls)) { | ||
912 | $allowed = false; | ||
913 | foreach ($options->allowed_urls as $allowurl) { | ||
914 | if (stristr($url, $allowurl) !== false) { | ||
915 | $allowed = true; | ||
916 | break; | ||
917 | } | ||
918 | } | ||
919 | if (!$allowed) return false; | ||
920 | } else { | ||
921 | foreach ($options->blocked_urls as $blockurl) { | ||
922 | if (stristr($url, $blockurl) !== false) { | ||
923 | return false; | ||
924 | } | ||
925 | } | ||
926 | } | ||
927 | return true; | ||
928 | } | ||
929 | |||
930 | ////////////////////////////////////////////// | ||
931 | // Convert $html to UTF8 | ||
932 | // (uses HTTP headers and HTML to find encoding) | ||
933 | // adapted from http://stackoverflow.com/questions/910793/php-detect-encoding-and-make-everything-utf-8 | ||
934 | ////////////////////////////////////////////// | ||
935 | function convert_to_utf8($html, $header=null) | ||
936 | { | ||
937 | $encoding = null; | ||
938 | if ($html || $header) { | ||
939 | if (is_array($header)) $header = implode("\n", $header); | ||
940 | if (!$header || !preg_match_all('/^Content-Type:\s+([^;]+)(?:;\s*charset=["\']?([^;"\'\n]*))?/im', $header, $match, PREG_SET_ORDER)) { | ||
941 | // error parsing the response | ||
942 | debug('Could not find Content-Type header in HTTP response'); | ||
943 | } else { | ||
944 | $match = end($match); // get last matched element (in case of redirects) | ||
945 | if (isset($match[2])) $encoding = trim($match[2], "\"' \r\n\0\x0B\t"); | ||
946 | } | ||
947 | // TODO: check to see if encoding is supported (can we convert it?) | ||
948 | // If it's not, result will be empty string. | ||
949 | // For now we'll check for invalid encoding types returned by some sites, e.g. 'none' | ||
950 | // Problem URL: http://facta.co.jp/blog/archives/20111026001026.html | ||
951 | if (!$encoding || $encoding == 'none') { | ||
952 | // search for encoding in HTML - only look at the first 50000 characters | ||
953 | // Why 50000? See, for example, http://www.lemonde.fr/festival-de-cannes/article/2012/05/23/deux-cretes-en-goguette-sur-la-croisette_1705732_766360.html | ||
954 | // TODO: improve this so it looks at smaller chunks first | ||
955 | $html_head = substr($html, 0, 50000); | ||
956 | if (preg_match('/^<\?xml\s+version=(?:"[^"]*"|\'[^\']*\')\s+encoding=("[^"]*"|\'[^\']*\')/s', $html_head, $match)) { | ||
957 | $encoding = trim($match[1], '"\''); | ||
958 | } elseif (preg_match('/<meta\s+http-equiv=["\']?Content-Type["\']? content=["\'][^;]+;\s*charset=["\']?([^;"\'>]+)/i', $html_head, $match)) { | ||
959 | $encoding = trim($match[1]); | ||
960 | } elseif (preg_match_all('/<meta\s+([^>]+)>/i', $html_head, $match)) { | ||
961 | foreach ($match[1] as $_test) { | ||
962 | if (preg_match('/charset=["\']?([^"\']+)/i', $_test, $_m)) { | ||
963 | $encoding = trim($_m[1]); | ||
964 | break; | ||
965 | } | ||
966 | } | ||
967 | } | ||
968 | } | ||
969 | if (isset($encoding)) $encoding = trim($encoding); | ||
970 | // trim is important here! | ||
971 | if (!$encoding || (strtolower($encoding) == 'iso-8859-1')) { | ||
972 | // replace MS Word smart qutoes | ||
973 | $trans = array(); | ||
974 | $trans[chr(130)] = '‚'; // Single Low-9 Quotation Mark | ||
975 | $trans[chr(131)] = 'ƒ'; // Latin Small Letter F With Hook | ||
976 | $trans[chr(132)] = '„'; // Double Low-9 Quotation Mark | ||
977 | $trans[chr(133)] = '…'; // Horizontal Ellipsis | ||
978 | $trans[chr(134)] = '†'; // Dagger | ||
979 | $trans[chr(135)] = '‡'; // Double Dagger | ||
980 | $trans[chr(136)] = 'ˆ'; // Modifier Letter Circumflex Accent | ||
981 | $trans[chr(137)] = '‰'; // Per Mille Sign | ||
982 | $trans[chr(138)] = 'Š'; // Latin Capital Letter S With Caron | ||
983 | $trans[chr(139)] = '‹'; // Single Left-Pointing Angle Quotation Mark | ||
984 | $trans[chr(140)] = 'Œ'; // Latin Capital Ligature OE | ||
985 | $trans[chr(145)] = '‘'; // Left Single Quotation Mark | ||
986 | $trans[chr(146)] = '’'; // Right Single Quotation Mark | ||
987 | $trans[chr(147)] = '“'; // Left Double Quotation Mark | ||
988 | $trans[chr(148)] = '”'; // Right Double Quotation Mark | ||
989 | $trans[chr(149)] = '•'; // Bullet | ||
990 | $trans[chr(150)] = '–'; // En Dash | ||
991 | $trans[chr(151)] = '—'; // Em Dash | ||
992 | $trans[chr(152)] = '˜'; // Small Tilde | ||
993 | $trans[chr(153)] = '™'; // Trade Mark Sign | ||
994 | $trans[chr(154)] = 'š'; // Latin Small Letter S With Caron | ||
995 | $trans[chr(155)] = '›'; // Single Right-Pointing Angle Quotation Mark | ||
996 | $trans[chr(156)] = 'œ'; // Latin Small Ligature OE | ||
997 | $trans[chr(159)] = 'Ÿ'; // Latin Capital Letter Y With Diaeresis | ||
998 | $html = strtr($html, $trans); | ||
999 | } | ||
1000 | if (!$encoding) { | ||
1001 | debug('No character encoding found, so treating as UTF-8'); | ||
1002 | $encoding = 'utf-8'; | ||
1003 | } else { | ||
1004 | debug('Character encoding: '.$encoding); | ||
1005 | if (strtolower($encoding) != 'utf-8') { | ||
1006 | debug('Converting to UTF-8'); | ||
1007 | $html = SimplePie_Misc::change_encoding($html, $encoding, 'utf-8'); | ||
1008 | /* | ||
1009 | if (function_exists('iconv')) { | ||
1010 | // iconv appears to handle certain character encodings better than mb_convert_encoding | ||
1011 | $html = iconv($encoding, 'utf-8', $html); | ||
1012 | } else { | ||
1013 | $html = mb_convert_encoding($html, 'utf-8', $encoding); | ||
1014 | } | ||
1015 | */ | ||
1016 | } | ||
1017 | } | ||
1018 | } | ||
1019 | return $html; | ||
1020 | } | ||
1021 | |||
1022 | function makeAbsolute($base, $elem) { | ||
1023 | $base = new SimplePie_IRI($base); | ||
1024 | // remove '//' in URL path (used to prevent URLs from resolving properly) | ||
1025 | // TODO: check if this is still the case | ||
1026 | if (isset($base->path)) $base->path = preg_replace('!//+!', '/', $base->path); | ||
1027 | foreach(array('a'=>'href', 'img'=>'src') as $tag => $attr) { | ||
1028 | $elems = $elem->getElementsByTagName($tag); | ||
1029 | for ($i = $elems->length-1; $i >= 0; $i--) { | ||
1030 | $e = $elems->item($i); | ||
1031 | //$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); | ||
1032 | makeAbsoluteAttr($base, $e, $attr); | ||
1033 | } | ||
1034 | if (strtolower($elem->tagName) == $tag) makeAbsoluteAttr($base, $elem, $attr); | ||
1035 | } | ||
1036 | } | ||
1037 | function makeAbsoluteAttr($base, $e, $attr) { | ||
1038 | if ($e->hasAttribute($attr)) { | ||
1039 | // Trim leading and trailing white space. I don't really like this but | ||
1040 | // unfortunately it does appear on some sites. e.g. <img src=" /path/to/image.jpg" /> | ||
1041 | $url = trim(str_replace('%20', ' ', $e->getAttribute($attr))); | ||
1042 | $url = str_replace(' ', '%20', $url); | ||
1043 | if (!preg_match('!https?://!i', $url)) { | ||
1044 | if ($absolute = SimplePie_IRI::absolutize($base, $url)) { | ||
1045 | $e->setAttribute($attr, $absolute); | ||
1046 | } | ||
1047 | } | ||
1048 | } | ||
1049 | } | ||
1050 | function makeAbsoluteStr($base, $url) { | ||
1051 | $base = new SimplePie_IRI($base); | ||
1052 | // remove '//' in URL path (causes URLs not to resolve properly) | ||
1053 | if (isset($base->path)) $base->path = preg_replace('!//+!', '/', $base->path); | ||
1054 | if (preg_match('!^https?://!i', $url)) { | ||
1055 | // already absolute | ||
1056 | return $url; | ||
1057 | } else { | ||
1058 | if ($absolute = SimplePie_IRI::absolutize($base, $url)) { | ||
1059 | return $absolute; | ||
1060 | } | ||
1061 | return false; | ||
1062 | } | ||
1063 | } | ||
1064 | // returns single page response, or false if not found | ||
1065 | function getSinglePage($item, $html, $url) { | ||
1066 | global $http, $extractor; | ||
1067 | debug('Looking for site config files to see if single page link exists'); | ||
1068 | $site_config = $extractor->buildSiteConfig($url, $html); | ||
1069 | $splink = null; | ||
1070 | if (!empty($site_config->single_page_link)) { | ||
1071 | $splink = $site_config->single_page_link; | ||
1072 | } elseif (!empty($site_config->single_page_link_in_feed)) { | ||
1073 | // single page link xpath is targeted at feed | ||
1074 | $splink = $site_config->single_page_link_in_feed; | ||
1075 | // so let's replace HTML with feed item description | ||
1076 | $html = $item->get_description(); | ||
1077 | } | ||
1078 | if (isset($splink)) { | ||
1079 | // Build DOM tree from HTML | ||
1080 | $readability = new Readability($html, $url); | ||
1081 | $xpath = new DOMXPath($readability->dom); | ||
1082 | // Loop through single_page_link xpath expressions | ||
1083 | $single_page_url = null; | ||
1084 | foreach ($splink as $pattern) { | ||
1085 | $elems = @$xpath->evaluate($pattern, $readability->dom); | ||
1086 | if (is_string($elems)) { | ||
1087 | $single_page_url = trim($elems); | ||
1088 | break; | ||
1089 | } elseif ($elems instanceof DOMNodeList && $elems->length > 0) { | ||
1090 | foreach ($elems as $item) { | ||
1091 | if ($item instanceof DOMElement && $item->hasAttribute('href')) { | ||
1092 | $single_page_url = $item->getAttribute('href'); | ||
1093 | break 2; | ||
1094 | } elseif ($item instanceof DOMAttr && $item->value) { | ||
1095 | $single_page_url = $item->value; | ||
1096 | break 2; | ||
1097 | } | ||
1098 | } | ||
1099 | } | ||
1100 | } | ||
1101 | // If we've got URL, resolve against $url | ||
1102 | if (isset($single_page_url) && ($single_page_url = makeAbsoluteStr($url, $single_page_url))) { | ||
1103 | // check it's not what we have already! | ||
1104 | if ($single_page_url != $url) { | ||
1105 | // it's not, so let's try to fetch it... | ||
1106 | $_prev_ref = $http->referer; | ||
1107 | $http->referer = $single_page_url; | ||
1108 | if (($response = $http->get($single_page_url, true)) && $response['status_code'] < 300) { | ||
1109 | $http->referer = $_prev_ref; | ||
1110 | return $response; | ||
1111 | } | ||
1112 | $http->referer = $_prev_ref; | ||
1113 | } | ||
1114 | } | ||
1115 | } | ||
1116 | return false; | ||
1117 | } | ||
1118 | |||
1119 | // based on content-type http header, decide what to do | ||
1120 | // param: HTTP headers string | ||
1121 | // return: array with keys: 'mime', 'type', 'subtype', 'action', 'name' | ||
1122 | // e.g. array('mime'=>'image/jpeg', 'type'=>'image', 'subtype'=>'jpeg', 'action'=>'link', 'name'=>'Image') | ||
1123 | function get_mime_action_info($headers) { | ||
1124 | global $options; | ||
1125 | // check if action defined for returned Content-Type | ||
1126 | $info = array(); | ||
1127 | if (preg_match('!^Content-Type:\s*(([-\w]+)/([-\w\+]+))!im', $headers, $match)) { | ||
1128 | // look for full mime type (e.g. image/jpeg) or just type (e.g. image) | ||
1129 | // match[1] = full mime type, e.g. image/jpeg | ||
1130 | // match[2] = first part, e.g. image | ||
1131 | // match[3] = last part, e.g. jpeg | ||
1132 | $info['mime'] = strtolower(trim($match[1])); | ||
1133 | $info['type'] = strtolower(trim($match[2])); | ||
1134 | $info['subtype'] = strtolower(trim($match[3])); | ||
1135 | foreach (array($info['mime'], $info['type']) as $_mime) { | ||
1136 | if (isset($options->content_type_exc[$_mime])) { | ||
1137 | $info['action'] = $options->content_type_exc[$_mime]['action']; | ||
1138 | $info['name'] = $options->content_type_exc[$_mime]['name']; | ||
1139 | break; | ||
1140 | } | ||
1141 | } | ||
1142 | } | ||
1143 | return $info; | ||
1144 | } | ||
1145 | |||
1146 | function remove_url_cruft($url) { | ||
1147 | // remove google analytics for the time being | ||
1148 | // regex adapted from http://navitronic.co.uk/2010/12/removing-google-analytics-cruft-from-urls/ | ||
1149 | // https://gist.github.com/758177 | ||
1150 | return preg_replace('/(\?|\&)utm_[a-z]+=[^\&]+/', '', $url); | ||
1151 | } | ||
1152 | |||
1153 | function make_substitutions($string) { | ||
1154 | if ($string == '') return $string; | ||
1155 | global $item, $effective_url; | ||
1156 | $string = str_replace('{url}', htmlspecialchars($item->get_permalink()), $string); | ||
1157 | $string = str_replace('{effective-url}', htmlspecialchars($effective_url), $string); | ||
1158 | return $string; | ||
1159 | } | ||
1160 | |||
1161 | function get_cache() { | ||
1162 | global $options, $valid_key; | ||
1163 | static $cache = null; | ||
1164 | if ($cache === null) { | ||
1165 | $frontendOptions = array( | ||
1166 | 'lifetime' => 10*60, // cache lifetime of 10 minutes | ||
1167 | 'automatic_serialization' => false, | ||
1168 | 'write_control' => false, | ||
1169 | 'automatic_cleaning_factor' => $options->cache_cleanup, | ||
1170 | 'ignore_user_abort' => false | ||
1171 | ); | ||
1172 | $backendOptions = array( | ||
1173 | 'cache_dir' => ($valid_key) ? $options->cache_dir.'/rss-with-key/' : $options->cache_dir.'/rss/', // directory where to put the cache files | ||
1174 | 'file_locking' => false, | ||
1175 | 'read_control' => true, | ||
1176 | 'read_control_type' => 'strlen', | ||
1177 | 'hashed_directory_level' => $options->cache_directory_level, | ||
1178 | 'hashed_directory_perm' => 0777, | ||
1179 | 'cache_file_perm' => 0664, | ||
1180 | 'file_name_prefix' => 'ff' | ||
1181 | ); | ||
1182 | // getting a Zend_Cache_Core object | ||
1183 | $cache = Zend_Cache::factory('Core', 'File', $frontendOptions, $backendOptions); | ||
1184 | } | ||
1185 | return $cache; | ||
1186 | } | ||
1187 | |||
1188 | function debug($msg) { | ||
1189 | global $debug_mode; | ||
1190 | if ($debug_mode) { | ||
1191 | echo '* ',$msg,"\n"; | ||
1192 | ob_flush(); | ||
1193 | flush(); | ||
1194 | } | ||
1195 | } \ No newline at end of file | ||