]> git.immae.eu Git - github/wallabag/wallabag.git/blame - inc/3rdparty/makefulltextfeed.php
projects / github / wallabag / wallabag.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Full-Text RSS included as a script instead of file_get_contents call. Tnx to @Faless...
[github/wallabag/wallabag.git] / inc / 3rdparty / makefulltextfeed.php
Content-type: text/html ]> git.immae.eu Git - github/wallabag/wallabag.git/blame - inc/3rdparty/makefulltextfeed.php
projects / github / wallabag / wallabag.git / blame
? search:


500 - Internal Server Error
Malformed UTF-8 character (fatal) at (eval 6) line 1, <$fd> line 1637.
A self hostable application for saving web pages: Save and classify articles. Read them later. Freely.
CommitLineData
42c80841
NL		 1<?php\r
2// Full-Text RSS: Create Full-Text Feeds\r
3// Author: Keyvan Minoukadeh\r
4// Copyright (c) 2013 Keyvan Minoukadeh\r
5// License: AGPLv3\r
6// Version: 3.1\r
7// Date: 2013-03-05\r
8// More info: http://fivefilters.org/content-only/\r
9// Help: http://help.fivefilters.org\r
10\r
11/*\r
12This program is free software: you can redistribute it and/or modify\r
13it under the terms of the GNU Affero General Public License as published by\r
14the Free Software Foundation, either version 3 of the License, or\r
15(at your option) any later version.\r
16\r
17This program is distributed in the hope that it will be useful,\r
18but WITHOUT ANY WARRANTY; without even the implied warranty of\r
19MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\r
20GNU Affero General Public License for more details.\r
21\r
22You should have received a copy of the GNU Affero General Public License\r
23along with this program. If not, see <http://www.gnu.org/licenses/>.\r
24*/\r
25\r
26// Usage\r
27// -----\r
28// Request this file passing it your feed in the querystring: makefulltextfeed.php?url=mysite.org\r
29// The following options can be passed in the querystring:\r
30// * URL: url=[feed or website url] (required, should be URL-encoded - in php: urlencode($url))\r
31// * URL points to HTML (not feed): html=true (optional, by default it's automatically detected)\r
32// * API key: key=[api key] (optional, refer to config.php)\r
33// * Max entries to process: max=[max number of items] (optional)\r
34\r
35error_reporting(E_ALL ^ E_NOTICE);\r
36ini_set("display_errors", 1);\r
37@set_time_limit(120);\r
38\r
39// Deal with magic quotes\r
40if (get_magic_quotes_gpc()) {\r
41 $process = array(&$_GET, &$_POST, &$_REQUEST);\r
42 while (list($key, $val) = each($process)) {\r
43 foreach ($val as $k => $v) {\r
44 unset($process[$key][$k]);\r
45 if (is_array($v)) {\r
46 $process[$key][stripslashes($k)] = $v;\r
47 $process[] = &$process[$key][stripslashes($k)];\r
48 } else {\r
49 $process[$key][stripslashes($k)] = stripslashes($v);\r
50 }\r
51 }\r
52 }\r
53 unset($process);\r
54}\r
55\r
56// set include path\r
57set_include_path(realpath(dirname(__FILE__).'/libraries').PATH_SEPARATOR.get_include_path());\r
58// Autoloading of classes allows us to include files only when they're\r
59// needed. If we've got a cached copy, for example, only Zend_Cache is loaded.\r
60function autoload($class_name) {\r
61 static $dir = null;\r
62 if ($dir === null) $dir = dirname(__FILE__).'/libraries/';\r
63 static $mapping = array(\r
64 // Include FeedCreator for RSS/Atom creation\r
65 'FeedWriter' => 'feedwriter/FeedWriter.php',\r
66 'FeedItem' => 'feedwriter/FeedItem.php',\r
67 // Include ContentExtractor and Readability for identifying and extracting content from URLs\r
68 'ContentExtractor' => 'content-extractor/ContentExtractor.php',\r
69 'SiteConfig' => 'content-extractor/SiteConfig.php',\r
70 'Readability' => 'readability/Readability.php',\r
71 // Include Humble HTTP Agent to allow parallel requests and response caching\r
72 'HumbleHttpAgent' => 'humble-http-agent/HumbleHttpAgent.php',\r
73 'SimplePie_HumbleHttpAgent' => 'humble-http-agent/SimplePie_HumbleHttpAgent.php',\r
74 'CookieJar' => 'humble-http-agent/CookieJar.php',\r
75 // Include Zend Cache to improve performance (cache results)\r
76 'Zend_Cache' => 'Zend/Cache.php',\r
77 // Language detect\r
78 'Text_LanguageDetect' => 'language-detect/LanguageDetect.php',\r
79 // HTML5 Lib\r
80 'HTML5_Parser' => 'html5/Parser.php',\r
81 // htmLawed - used if XSS filter is enabled (xss_filter)\r
82 'htmLawed' => 'htmLawed/htmLawed.php'\r
83 );\r
84 if (isset($mapping[$class_name])) {\r
85 debug("** Loading class $class_name ({$mapping[$class_name]})");\r
86 require $dir.$mapping[$class_name];\r
87 return true;\r
88 } else {\r
89 return false;\r
90 }\r
91}\r
92spl_autoload_register('autoload');\r
93require dirname(__FILE__).'/libraries/simplepie/autoloader.php';\r
94\r
95////////////////////////////////\r
96// Load config file\r
97////////////////////////////////\r
98require dirname(__FILE__).'/config.php';\r
99\r
100////////////////////////////////\r
101// Prevent indexing/following by search engines because:\r
102// 1. The content is already public and presumably indexed (why create duplicates?)\r
103// 2. Not doing so might increase number of requests from search engines, thus increasing server load\r
104// Note: feed readers and services such as Yahoo Pipes will not be affected by this header.\r
105// Note: Using Disallow in a robots.txt file will be more effective (search engines will check\r
106// that before even requesting makefulltextfeed.php).\r
107////////////////////////////////\r
108header('X-Robots-Tag: noindex, nofollow');\r
109\r
110////////////////////////////////\r
111// Check if service is enabled\r
112////////////////////////////////\r
113if (!$options->enabled) { \r
114 die('The full-text RSS service is currently disabled'); \r
115}\r
116\r
117////////////////////////////////\r
118// Debug mode?\r
119// See the config file for debug options.\r
120////////////////////////////////\r
121$debug_mode = false;\r
122if (isset($_GET['debug'])) {\r
123 if ($options->debug === true || $options->debug == 'user') {\r
124 $debug_mode = true;\r
125 } elseif ($options->debug == 'admin') {\r
126 session_start();\r
127 $debug_mode = (@$_SESSION['auth'] == 1);\r
128 }\r
129 if ($debug_mode) {\r
130 header('Content-Type: text/plain; charset=utf-8');\r
131 } else {\r
132 if ($options->debug == 'admin') {\r
133 die('You must be logged in to the <a href="admin/">admin area</a> to see debug output.');\r
134 } else {\r
135 die('Debugging is disabled.');\r
136 }\r
137 }\r
138}\r
139\r
140////////////////////////////////\r
141// Check for APC\r
142////////////////////////////////\r
143$options->apc = $options->apc && function_exists('apc_add');\r
144if ($options->apc) {\r
145 debug('APC is enabled and available on server');\r
146} else {\r
147 debug('APC is disabled or not available on server');\r
148}\r
149\r
150////////////////////////////////\r
151// Check for smart cache\r
152////////////////////////////////\r
153$options->smart_cache = $options->smart_cache && function_exists('apc_inc');\r
154\r
155////////////////////////////////\r
156// Check for feed URL\r
157////////////////////////////////\r
158if (!isset($_GET['url'])) { \r
159 die('No URL supplied'); \r
160}\r
161$url = trim($_GET['url']);\r
162if (strtolower(substr($url, 0, 7)) == 'feed://') {\r
163 $url = 'http://'.substr($url, 7);\r
164}\r
165if (!preg_match('!^https?://.+!i', $url)) {\r
166 $url = 'http://'.$url;\r
167}\r
168\r
169$url = filter_var($url, FILTER_SANITIZE_URL);\r
170$test = filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);\r
171// deal with bug http://bugs.php.net/51192 (present in PHP 5.2.13 and PHP 5.3.2)\r
172if ($test === false) {\r
173 $test = filter_var(strtr($url, '-', '_'), FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED);\r
174}\r
175if ($test !== false && $test !== null && preg_match('!^https?://!', $url)) {\r
176 // all okay\r
177 unset($test);\r
178} else {\r
179 die('Invalid URL supplied');\r
180}\r
181debug("Supplied URL: $url");\r
182\r
183/////////////////////////////////\r
184// Redirect to hide API key\r
185/////////////////////////////////\r
186if (isset($_GET['key']) && ($key_index = array_search($_GET['key'], $options->api_keys)) !== false) {\r
187 $host = $_SERVER['HTTP_HOST'];\r
188 $path = rtrim(dirname($_SERVER['SCRIPT_NAME']), '/\\');\r
189 $_qs_url = (strtolower(substr($url, 0, 7)) == 'http://') ? substr($url, 7) : $url;\r
190 $redirect = 'http://'.htmlspecialchars($host.$path).'/makefulltextfeed.php?url='.urlencode($_qs_url);\r
191 $redirect .= '&key='.$key_index;\r
192 $redirect .= '&hash='.urlencode(sha1($_GET['key'].$url));\r
193 if (isset($_GET['html'])) $redirect .= '&html='.urlencode($_GET['html']);\r
194 if (isset($_GET['max'])) $redirect .= '&max='.(int)$_GET['max'];\r
195 if (isset($_GET['links'])) $redirect .= '&links='.urlencode($_GET['links']);\r
196 if (isset($_GET['exc'])) $redirect .= '&exc='.urlencode($_GET['exc']);\r
197 if (isset($_GET['format'])) $redirect .= '&format='.urlencode($_GET['format']);\r
198 if (isset($_GET['callback'])) $redirect .= '&callback='.urlencode($_GET['callback']); \r
199 if (isset($_GET['l'])) $redirect .= '&l='.urlencode($_GET['l']);\r
200 if (isset($_GET['xss'])) $redirect .= '&xss';\r
201 if (isset($_GET['use_extracted_title'])) $redirect .= '&use_extracted_title';\r
202 if (isset($_GET['debug'])) $redirect .= '&debug';\r
203 if ($debug_mode) {\r
204 debug('Redirecting to hide access key, follow URL below to continue');\r
205 debug("Location: $redirect");\r
206 } else {\r
207 header("Location: $redirect");\r
208 }\r
209 exit;\r
210}\r
211\r
212///////////////////////////////////////////////\r
213// Set timezone.\r
214// Prevents warnings, but needs more testing - \r
215// perhaps if timezone is set in php.ini we\r
216// don't need to set it at all...\r
217///////////////////////////////////////////////\r
218if (!ini_get('date.timezone') || !@date_default_timezone_set(ini_get('date.timezone'))) {\r
219 date_default_timezone_set('UTC');\r
220}\r
221\r
222///////////////////////////////////////////////\r
223// Check if the request is explicitly for an HTML page\r
224///////////////////////////////////////////////\r
225$html_only = (isset($_GET['html']) && ($_GET['html'] == '1' || $_GET['html'] == 'true'));\r
226\r
227///////////////////////////////////////////////\r
228// Check if valid key supplied\r
229///////////////////////////////////////////////\r
230$valid_key = false;\r
231if (isset($_GET['key']) && isset($_GET['hash']) && isset($options->api_keys[(int)$_GET['key']])) {\r
232 $valid_key = ($_GET['hash'] == sha1($options->api_keys[(int)$_GET['key']].$url));\r
233}\r
234$key_index = ($valid_key) ? (int)$_GET['key'] : 0;\r
235if (!$valid_key && $options->key_required) {\r
236 die('A valid key must be supplied'); \r
237}\r
238if (!$valid_key && isset($_GET['key']) && $_GET['key'] != '') {\r
239 die('The entered key is invalid');\r
240}\r
241\r
242if (file_exists('custom_init.php')) require 'custom_init.php';\r
243\r
244///////////////////////////////////////////////\r
245// Check URL against list of blacklisted URLs\r
246///////////////////////////////////////////////\r
247if (!url_allowed($url)) die('URL blocked');\r
248\r
249///////////////////////////////////////////////\r
250// Max entries\r
251// see config.php to find these values\r
252///////////////////////////////////////////////\r
253if (isset($_GET['max'])) {\r
254 $max = (int)$_GET['max'];\r
255 if ($valid_key) {\r
256 $max = min($max, $options->max_entries_with_key);\r
257 } else {\r
258 $max = min($max, $options->max_entries);\r
259 }\r
260} else {\r
261 if ($valid_key) {\r
262 $max = $options->default_entries_with_key;\r
263 } else {\r
264 $max = $options->default_entries;\r
265 }\r
266}\r
267\r
268///////////////////////////////////////////////\r
269// Link handling\r
270///////////////////////////////////////////////\r
271if (isset($_GET['links']) && in_array($_GET['links'], array('preserve', 'footnotes', 'remove'))) {\r
272 $links = $_GET['links'];\r
273} else {\r
274 $links = 'preserve';\r
275}\r
276\r
277///////////////////////////////////////////////\r
278// Favour item titles in feed?\r
279///////////////////////////////////////////////\r
280$favour_feed_titles = true;\r
281if ($options->favour_feed_titles == 'user') {\r
282 $favour_feed_titles = !isset($_GET['use_extracted_title']);\r
283} else {\r
284 $favour_feed_titles = $options->favour_feed_titles;\r
285}\r
286\r
287///////////////////////////////////////////////\r
288// Exclude items if extraction fails\r
289///////////////////////////////////////////////\r
290if ($options->exclude_items_on_fail === 'user') {\r
291 $exclude_on_fail = (isset($_GET['exc']) && ($_GET['exc'] == '1'));\r
292} else {\r
293 $exclude_on_fail = $options->exclude_items_on_fail;\r
294}\r
295\r
296///////////////////////////////////////////////\r
297// Detect language\r
298///////////////////////////////////////////////\r
299if ($options->detect_language === 'user') {\r
300 if (isset($_GET['l'])) {\r
301 $detect_language = (int)$_GET['l'];\r
302 } else {\r
303 $detect_language = 1;\r
304 }\r
305} else {\r
306 $detect_language = $options->detect_language;\r
307}\r
308\r
309if ($detect_language >= 2) {\r
310 $language_codes = array('albanian' => 'sq','arabic' => 'ar','azeri' => 'az','bengali' => 'bn','bulgarian' => 'bg',\r
311 'cebuano' => 'ceb', // ISO 639-2\r
312 'croatian' => 'hr','czech' => 'cs','danish' => 'da','dutch' => 'nl','english' => 'en','estonian' => 'et','farsi' => 'fa','finnish' => 'fi','french' => 'fr','german' => 'de','hausa' => 'ha',\r
313 'hawaiian' => 'haw', // ISO 639-2 \r
314 'hindi' => 'hi','hungarian' => 'hu','icelandic' => 'is','indonesian' => 'id','italian' => 'it','kazakh' => 'kk','kyrgyz' => 'ky','latin' => 'la','latvian' => 'lv','lithuanian' => 'lt','macedonian' => 'mk','mongolian' => 'mn','nepali' => 'ne','norwegian' => 'no','pashto' => 'ps',\r
315 'pidgin' => 'cpe', // ISO 639-2 \r
316 'polish' => 'pl','portuguese' => 'pt','romanian' => 'ro','russian' => 'ru','serbian' => 'sr','slovak' => 'sk','slovene' => 'sl','somali' => 'so','spanish' => 'es','swahili' => 'sw','swedish' => 'sv','tagalog' => 'tl','turkish' => 'tr','ukrainian' => 'uk','urdu' => 'ur','uzbek' => 'uz','vietnamese' => 'vi','welsh' => 'cy');\r
317}\r
318$use_cld = extension_loaded('cld') && (version_compare(PHP_VERSION, '5.3.0') >= 0);\r
319\r
320/////////////////////////////////////\r
321// Check for valid format\r
322// (stick to RSS (or RSS as JSON) for the time being)\r
323/////////////////////////////////////\r
324if (isset($_GET['format']) && $_GET['format'] == 'json') {\r
325 $format = 'json';\r
326} else {\r
327 $format = 'rss';\r
328}\r
329\r
330/////////////////////////////////////\r
331// Should we do XSS filtering?\r
332/////////////////////////////////////\r
333if ($options->xss_filter === 'user') {\r
334 $xss_filter = isset($_GET['xss']);\r
335} else {\r
336 $xss_filter = $options->xss_filter;\r
337}\r
338if (!$xss_filter && isset($_GET['xss'])) {\r
339 die('XSS filtering is disabled in config');\r
340}\r
341\r
342/////////////////////////////////////\r
343// Check for JSONP\r
344// Regex from https://gist.github.com/1217080\r
345/////////////////////////////////////\r
346$callback = null;\r
347if ($format =='json' && isset($_GET['callback'])) {\r
348 $callback = trim($_GET['callback']);\r
349 foreach (explode('.', $callback) as $_identifier) {\r
350 if (!preg_match('/^[a-zA-Z_$][0-9a-zA-Z_$]*(?:\[(?:".+"|\'.+\'|\d+)\])*?$/', $_identifier)) {\r
351 die('Invalid JSONP callback');\r
352 }\r
353 }\r
354 debug("JSONP callback: $callback");\r
355}\r
356\r
357//////////////////////////////////\r
358// Enable Cross-Origin Resource Sharing (CORS)\r
359//////////////////////////////////\r
360if ($options->cors) header('Access-Control-Allow-Origin: *');\r
361\r
362//////////////////////////////////\r
363// Check for cached copy\r
364//////////////////////////////////\r
365if ($options->caching) {\r
366 debug('Caching is enabled...');\r
367 $cache_id = md5($max.$url.$valid_key.$links.$favour_feed_titles.$xss_filter.$exclude_on_fail.$format.$detect_language.(int)isset($_GET['pubsub']));\r
368 $check_cache = true;\r
369 if ($options->apc && $options->smart_cache) {\r
370 apc_add("cache.$cache_id", 0, 10*60);\r
371 $apc_cache_hits = (int)apc_fetch("cache.$cache_id");\r
372 $check_cache = ($apc_cache_hits >= 2);\r
373 apc_inc("cache.$cache_id");\r
374 if ($check_cache) {\r
375 debug('Cache key found in APC, we\'ll try to load cache file from disk');\r
376 } else {\r
377 debug('Cache key not found in APC');\r
378 }\r
379 }\r
380 if ($check_cache) {\r
381 $cache = get_cache();\r
382 if ($data = $cache->load($cache_id)) {\r
383 if ($debug_mode) {\r
384 debug('Loaded cached copy');\r
385 exit;\r
386 }\r
387 if ($format == 'json') {\r
388 if ($callback === null) {\r
389 header('Content-type: application/json; charset=UTF-8');\r
390 } else {\r
391 header('Content-type: application/javascript; charset=UTF-8');\r
392 }\r
393 } else {\r
394 header('Content-type: text/xml; charset=UTF-8');\r
395 header('X-content-type-options: nosniff');\r
396 }\r
397 if (headers_sent()) die('Some data has already been output, can\'t send RSS file');\r
398 if ($callback) {\r
399 echo "$callback($data);";\r
400 } else {\r
401 echo $data;\r
402 }\r
403 exit;\r
404 }\r
405 }\r
406}\r
407\r
408//////////////////////////////////\r
409// Set Expires header\r
410//////////////////////////////////\r
411if (!$debug_mode) {\r
412 header('Expires: ' . gmdate('D, d M Y H:i:s', time()+(60*10)) . ' GMT');\r
413}\r
414\r
415//////////////////////////////////\r
416// Set up HTTP agent\r
417//////////////////////////////////\r
418$http = new HumbleHttpAgent();\r
419$http->debug = $debug_mode;\r
420$http->userAgentMap = $options->user_agents;\r
421$http->headerOnlyTypes = array_keys($options->content_type_exc);\r
422$http->rewriteUrls = $options->rewrite_url;\r
423\r
424//////////////////////////////////\r
425// Set up Content Extractor\r
426//////////////////////////////////\r
b4fd2154 427global $extractor;\r
42c80841
NL		 428$extractor = new ContentExtractor(dirname(__FILE__).'/site_config/custom', dirname(__FILE__).'/site_config/standard');\r
429$extractor->debug = $debug_mode;\r
430SiteConfig::$debug = $debug_mode;\r
431SiteConfig::use_apc($options->apc);\r
432$extractor->fingerprints = $options->fingerprints;\r
433$extractor->allowedParsers = $options->allowed_parsers;\r
434\r
435////////////////////////////////\r
436// Get RSS/Atom feed\r
437////////////////////////////////\r
438if (!$html_only) {\r
439 debug('--------');\r
440 debug("Attempting to process URL as feed");\r
441 // Send user agent header showing PHP (prevents a HTML response from feedburner)\r
442 $http->userAgentDefault = HumbleHttpAgent::UA_PHP;\r
443 // configure SimplePie HTTP extension class to use our HumbleHttpAgent instance\r
444 SimplePie_HumbleHttpAgent::set_agent($http);\r
445 $feed = new SimplePie();\r
446 // some feeds use the text/html content type - force_feed tells SimplePie to process anyway\r
447 $feed->force_feed(true);\r
448 $feed->set_file_class('SimplePie_HumbleHttpAgent');\r
449 //$feed->set_feed_url($url); // colons appearing in the URL's path get encoded\r
450 $feed->feed_url = $url;\r
451 $feed->set_autodiscovery_level(SIMPLEPIE_LOCATOR_NONE);\r
452 $feed->set_timeout(20);\r
453 $feed->enable_cache(false);\r
454 $feed->set_stupidly_fast(true);\r
455 $feed->enable_order_by_date(false); // we don't want to do anything to the feed\r
456 $feed->set_url_replacements(array());\r
457 // initialise the feed\r
458 // the @ suppresses notices which on some servers causes a 500 internal server error\r
459 $result = @$feed->init();\r
460 //$feed->handle_content_type();\r
461 //$feed->get_title();\r
462 if ($result && (!is_array($feed->data) || count($feed->data) == 0)) {\r
463 die('Sorry, no feed items found');\r
464 }\r
465 // from now on, we'll identify ourselves as a browser\r
466 $http->userAgentDefault = HumbleHttpAgent::UA_BROWSER;\r
467}\r
468\r
469////////////////////////////////////////////////////////////////////////////////\r
470// Our given URL is not a feed, so let's create our own feed with a single item:\r
471// the given URL. This basically treats all non-feed URLs as if they were\r
472// single-item feeds.\r
473////////////////////////////////////////////////////////////////////////////////\r
474$isDummyFeed = false;\r
475if ($html_only || !$result) {\r
476 debug('--------');\r
477 debug("Constructing a single-item feed from URL");\r
478 $isDummyFeed = true;\r
479 unset($feed, $result);\r
480 // create single item dummy feed object\r
481 class DummySingleItemFeed {\r
482 public $item;\r
483 function __construct($url) { $this->item = new DummySingleItem($url); }\r
484 public function get_title() { return ''; }\r
485 public function get_description() { return 'Content extracted from '.$this->item->url; }\r
486 public function get_link() { return $this->item->url; }\r
487 public function get_language() { return false; }\r
488 public function get_image_url() { return false; }\r
489 public function get_items($start=0, $max=1) { return array(0=>$this->item); }\r
490 }\r
491 class DummySingleItem {\r
492 public $url;\r
493 function __construct($url) { $this->url = $url; }\r
494 public function get_permalink() { return $this->url; }\r
495 public function get_title() { return null; }\r
496 public function get_date($format='') { return false; }\r
497 public function get_author($key=0) { return null; }\r
498 public function get_authors() { return null; }\r
499 public function get_description() { return ''; }\r
500 public function get_enclosure($key=0, $prefer=null) { return null; }\r
501 public function get_enclosures() { return null; }\r
502 public function get_categories() { return null; }\r
503 }\r
504 $feed = new DummySingleItemFeed($url);\r
505}\r
506\r
507////////////////////////////////////////////\r
508// Create full-text feed\r
509////////////////////////////////////////////\r
510$output = new FeedWriter();\r
511$output->setTitle(strip_tags($feed->get_title()));\r
512$output->setDescription(strip_tags($feed->get_description()));\r
513$output->setXsl('css/feed.xsl'); // Chrome uses this, most browsers ignore it\r
514if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment\r
515 $output->addHub('http://fivefilters.superfeedr.com/');\r
516 $output->addHub('http://pubsubhubbub.appspot.com/');\r
517 $output->setSelf('http://'.$_SERVER['HTTP_HOST'].$_SERVER['REQUEST_URI']);\r
518}\r
519$output->setLink($feed->get_link()); // Google Reader uses this for pulling in favicons\r
520if ($img_url = $feed->get_image_url()) {\r
521 $output->setImage($feed->get_title(), $feed->get_link(), $img_url);\r
522}\r
523\r
524////////////////////////////////////////////\r
525// Loop through feed items\r
526////////////////////////////////////////////\r
527$items = $feed->get_items(0, $max); \r
528// Request all feed items in parallel (if supported)\r
529$urls_sanitized = array();\r
530$urls = array();\r
531foreach ($items as $key => $item) {\r
532 $permalink = htmlspecialchars_decode($item->get_permalink());\r
533 // Colons in URL path segments get encoded by SimplePie, yet some sites expect them unencoded\r
534 $permalink = str_replace('%3A', ':', $permalink);\r
535 // validateUrl() strips non-ascii characters\r
536 // simplepie already sanitizes URLs so let's not do it again here.\r
537 //$permalink = $http->validateUrl($permalink);\r
538 if ($permalink) {\r
539 $urls_sanitized[] = $permalink;\r
540 }\r
541 $urls[$key] = $permalink;\r
542}\r
543debug('--------');\r
544debug('Fetching feed items');\r
545$http->fetchAll($urls_sanitized);\r
546//$http->cacheAll();\r
547\r
548// count number of items added to full feed\r
549$item_count = 0;\r
550\r
551foreach ($items as $key => $item) {\r
552 debug('--------');\r
553 debug('Processing feed item '.($item_count+1));\r
554 $do_content_extraction = true;\r
555 $extract_result = false;\r
556 $text_sample = null;\r
557 $permalink = $urls[$key];\r
558 debug("Item URL: $permalink");\r
559 $extracted_title = '';\r
560 $feed_item_title = $item->get_title();\r
561 if ($feed_item_title !== null) {\r
562 $feed_item_title = strip_tags(htmlspecialchars_decode($feed_item_title));\r
563 }\r
564 $newitem = $output->createNewItem();\r
565 $newitem->setTitle($feed_item_title);\r
566 if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment\r
567 if ($permalink !== false) {\r
568 $newitem->setLink('http://fivefilters.org/content-only/redirect.php?url='.urlencode($permalink));\r
569 } else {\r
570 $newitem->setLink('http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()));\r
571 }\r
572 } else {\r
573 if ($permalink !== false) {\r
574 $newitem->setLink($permalink);\r
575 } else {\r
576 $newitem->setLink($item->get_permalink());\r
577 }\r
578 }\r
579 //if ($permalink && ($response = $http->get($permalink, true)) && $response['status_code'] < 300) {\r
580 // Allowing error codes - some sites return correct content with error status\r
581 // e.g. prospectmagazine.co.uk returns 403\r
582 if ($permalink && ($response = $http->get($permalink, true)) && ($response['status_code'] < 300 || $response['status_code'] > 400)) {\r
583 $effective_url = $response['effective_url'];\r
584 if (!url_allowed($effective_url)) continue;\r
585 // check if action defined for returned Content-Type\r
586 $mime_info = get_mime_action_info($response['headers']);\r
587 if (isset($mime_info['action'])) {\r
588 if ($mime_info['action'] == 'exclude') {\r
589 continue; // skip this feed item entry\r
590 } elseif ($mime_info['action'] == 'link') {\r
591 if ($mime_info['type'] == 'image') {\r
592 $html = "<a href=\"$effective_url\"><img src=\"$effective_url\" alt=\"{$mime_info['name']}\" /></a>";\r
593 } else {\r
594 $html = "<a href=\"$effective_url\">Download {$mime_info['name']}</a>";\r
595 }\r
596 $extracted_title = $mime_info['name'];\r
597 $do_content_extraction = false;\r
598 }\r
599 }\r
600 if ($do_content_extraction) {\r
601 $html = $response['body'];\r
602 // remove strange things\r
603 $html = str_replace('</[>', '', $html);\r
604 $html = convert_to_utf8($html, $response['headers']);\r
605 // check site config for single page URL - fetch it if found\r
606 $is_single_page = false;\r
607 if ($single_page_response = getSinglePage($item, $html, $effective_url)) {\r
608 $is_single_page = true;\r
609 $html = $single_page_response['body'];\r
610 // remove strange things\r
611 $html = str_replace('</[>', '', $html); \r
612 $html = convert_to_utf8($html, $single_page_response['headers']);\r
613 $effective_url = $single_page_response['effective_url'];\r
614 debug("Retrieved single-page view from $effective_url");\r
615 unset($single_page_response);\r
616 }\r
617 debug('--------');\r
618 debug('Attempting to extract content');\r
619 $extract_result = $extractor->process($html, $effective_url);\r
620 $readability = $extractor->readability;\r
621 $content_block = ($extract_result) ? $extractor->getContent() : null; \r
622 $extracted_title = ($extract_result) ? $extractor->getTitle() : '';\r
623 // Deal with multi-page articles\r
624 //die('Next: '.$extractor->getNextPageUrl());\r
625 $is_multi_page = (!$is_single_page && $extract_result && $extractor->getNextPageUrl());\r
626 if ($options->multipage && $is_multi_page) {\r
627 debug('--------');\r
628 debug('Attempting to process multi-page article');\r
629 $multi_page_urls = array();\r
630 $multi_page_content = array();\r
631 while ($next_page_url = $extractor->getNextPageUrl()) {\r
632 debug('--------');\r
633 debug('Processing next page: '.$next_page_url);\r
634 // If we've got URL, resolve against $url\r
635 if ($next_page_url = makeAbsoluteStr($effective_url, $next_page_url)) {\r
636 // check it's not what we have already!\r
637 if (!in_array($next_page_url, $multi_page_urls)) {\r
638 // it's not, so let's attempt to fetch it\r
639 $multi_page_urls[] = $next_page_url; \r
640 $_prev_ref = $http->referer;\r
641 if (($response = $http->get($next_page_url, true)) && $response['status_code'] < 300) {\r
642 // make sure mime type is not something with a different action associated\r
643 $page_mime_info = get_mime_action_info($response['headers']);\r
644 if (!isset($page_mime_info['action'])) {\r
645 $html = $response['body'];\r
646 // remove strange things\r
647 $html = str_replace('</[>', '', $html);\r
648 $html = convert_to_utf8($html, $response['headers']);\r
649 if ($extractor->process($html, $next_page_url)) {\r
650 $multi_page_content[] = $extractor->getContent();\r
651 continue;\r
652 } else { debug('Failed to extract content'); }\r
653 } else { debug('MIME type requires different action'); }\r
654 } else { debug('Failed to fetch URL'); }\r
655 } else { debug('URL already processed'); }\r
656 } else { debug('Failed to resolve against '.$effective_url); }\r
657 // failed to process next_page_url, so cancel further requests\r
658 $multi_page_content = array();\r
659 break;\r
660 }\r
661 // did we successfully deal with this multi-page article?\r
662 if (empty($multi_page_content)) {\r
663 debug('Failed to extract all parts of multi-page article, so not going to include them');\r
664 $multi_page_content[] = $readability->dom->createElement('p')->innerHTML = '<em>This article appears to continue on subsequent pages which we could not extract</em>';\r
665 }\r
666 foreach ($multi_page_content as $_page) {\r
667 $_page = $content_block->ownerDocument->importNode($_page, true);\r
668 $content_block->appendChild($_page);\r
669 }\r
670 unset($multi_page_urls, $multi_page_content, $page_mime_info, $next_page_url);\r
671 }\r
672 }\r
673 // use extracted title for both feed and item title if we're using single-item dummy feed\r
674 if ($isDummyFeed) {\r
675 $output->setTitle($extracted_title);\r
676 $newitem->setTitle($extracted_title);\r
677 } else {\r
678 // use extracted title instead of feed item title?\r
679 if (!$favour_feed_titles && $extracted_title != '') {\r
680 debug('Using extracted title in generated feed');\r
681 $newitem->setTitle($extracted_title);\r
682 }\r
683 }\r
684 }\r
685 if ($do_content_extraction) {\r
686 // if we failed to extract content...\r
687 if (!$extract_result) {\r
688 if ($exclude_on_fail) {\r
689 debug('Failed to extract, so skipping (due to exclude on fail parameter)');\r
690 continue; // skip this and move to next item\r
691 }\r
692 //TODO: get text sample for language detection\r
693 $html = $options->error_message;\r
694 // keep the original item description\r
695 $html .= $item->get_description();\r
696 } else {\r
697 $readability->clean($content_block, 'select');\r
698 if ($options->rewrite_relative_urls) makeAbsolute($effective_url, $content_block);\r
699 // footnotes\r
700 if (($links == 'footnotes') && (strpos($effective_url, 'wikipedia.org') === false)) {\r
701 $readability->addFootnotes($content_block);\r
702 }\r
703 // remove nesting: <div><div><div><p>test</p></div></div></div> = <p>test</p>\r
704 while ($content_block->childNodes->length == 1 && $content_block->firstChild->nodeType === XML_ELEMENT_NODE) {\r
705 // only follow these tag names\r
706 if (!in_array(strtolower($content_block->tagName), array('div', 'article', 'section', 'header', 'footer'))) break;\r
707 //$html = $content_block->firstChild->innerHTML; // FTR 2.9.5\r
708 $content_block = $content_block->firstChild;\r
709 }\r
710 // convert content block to HTML string\r
711 // Need to preserve things like body: //img[@id='feature']\r
712 if (in_array(strtolower($content_block->tagName), array('div', 'article', 'section', 'header', 'footer'))) {\r
713 $html = $content_block->innerHTML;\r
714 } else {\r
715 $html = $content_block->ownerDocument->saveXML($content_block); // essentially outerHTML\r
716 }\r
717 unset($content_block);\r
718 // post-processing cleanup\r
719 $html = preg_replace('!<p>[\s\h\v]*</p>!u', '', $html);\r
720 if ($links == 'remove') {\r
721 $html = preg_replace('!</?a[^>]*>!', '', $html);\r
722 }\r
723 // get text sample for language detection\r
724 $text_sample = strip_tags(substr($html, 0, 500));\r
725 $html = make_substitutions($options->message_to_prepend).$html;\r
726 $html .= make_substitutions($options->message_to_append);\r
727 }\r
728 }\r
729\r
730 if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment\r
731 $newitem->addElement('guid', 'http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()), array('isPermaLink'=>'false'));\r
732 } else {\r
733 $newitem->addElement('guid', $item->get_permalink(), array('isPermaLink'=>'true'));\r
734 }\r
735 // filter xss?\r
736 if ($xss_filter) {\r
737 debug('Filtering HTML to remove XSS');\r
738 $html = htmLawed::hl($html, array('safe'=>1, 'deny_attribute'=>'style', 'comment'=>1, 'cdata'=>1));\r
739 }\r
740 $newitem->setDescription($html);\r
741 \r
742 // set date\r
743 if ((int)$item->get_date('U') > 0) {\r
744 $newitem->setDate((int)$item->get_date('U'));\r
745 } elseif ($extractor->getDate()) {\r
746 $newitem->setDate($extractor->getDate());\r
747 }\r
748 \r
749 // add authors\r
750 if ($authors = $item->get_authors()) {\r
751 foreach ($authors as $author) {\r
752 // for some feeds, SimplePie stores author's name as email, e.g. http://feeds.feedburner.com/nymag/intel\r
753 if ($author->get_name() !== null) {\r
754 $newitem->addElement('dc:creator', $author->get_name());\r
755 } elseif ($author->get_email() !== null) {\r
756 $newitem->addElement('dc:creator', $author->get_email());\r
757 }\r
758 }\r
759 } elseif ($authors = $extractor->getAuthors()) {\r
760 //TODO: make sure the list size is reasonable\r
761 foreach ($authors as $author) {\r
762 // TODO: xpath often selects authors from other articles linked from the page.\r
763 // for now choose first item\r
764 $newitem->addElement('dc:creator', $author);\r
765 break;\r
766 }\r
767 }\r
768 \r
769 // add language\r
770 if ($detect_language) {\r
771 $language = $extractor->getLanguage();\r
772 if (!$language) $language = $feed->get_language();\r
773 if (($detect_language == 3 || (!$language && $detect_language == 2)) && $text_sample) {\r
774 try {\r
775 if ($use_cld) {\r
776 // Use PHP-CLD extension\r
777 $php_cld = 'CLD\detect'; // in quotes to prevent PHP 5.2 parse error\r
778 $res = $php_cld($text_sample);\r
779 if (is_array($res) && count($res) > 0) {\r
780 $language = $res[0]['code'];\r
781 } \r
782 } else {\r
783 //die('what');\r
784 // Use PEAR's Text_LanguageDetect\r
785 if (!isset($l)) {\r
786 $l = new Text_LanguageDetect('libraries/language-detect/lang.dat', 'libraries/language-detect/unicode_blocks.dat');\r
787 }\r
788 $l_result = $l->detect($text_sample, 1);\r
789 if (count($l_result) > 0) {\r
790 $language = $language_codes[key($l_result)];\r
791 }\r
792 }\r
793 } catch (Exception $e) {\r
794 //die('error: '.$e); \r
795 // do nothing\r
796 }\r
797 }\r
798 if ($language && (strlen($language) < 7)) { \r
799 $newitem->addElement('dc:language', $language);\r
800 }\r
801 }\r
802 \r
803 // add MIME type (if it appeared in our exclusions lists)\r
804 if (isset($mime_info['mime'])) $newitem->addElement('dc:format', $mime_info['mime']);\r
805 // add effective URL (URL after redirects)\r
806 if (isset($effective_url)) {\r
807 //TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g.\r
808