[\s\h\v]*
!u', '', $html); if ($links == 'remove') { @@ -727,130 +707,155 @@ foreach ($items as $key => $item) { } } - if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment - $newitem->addElement('guid', 'http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()), array('isPermaLink'=>'false')); + if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment + $newitem->addElement('guid', 'http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()), array('isPermaLink'=>'false')); + } else { + $newitem->addElement('guid', $item->get_permalink(), array('isPermaLink'=>'true')); + } + // filter xss? + if ($xss_filter) { + debug('Filtering HTML to remove XSS'); + $html = htmLawed::hl($html, array('safe'=>1, 'deny_attribute'=>'style', 'comment'=>1, 'cdata'=>1)); + } + + // add content + if ($options->summary === true) { + // get summary + $summary = ''; + if (!$do_content_extraction) { + $summary = $html; } else { - $newitem->addElement('guid', $item->get_permalink(), array('isPermaLink'=>'true')); - } - // filter xss? - if ($xss_filter) { - debug('Filtering HTML to remove XSS'); - $html = htmLawed::hl($html, array('safe'=>1, 'deny_attribute'=>'style', 'comment'=>1, 'cdata'=>1)); - } - $newitem->setDescription($html); - - // set date - if ((int)$item->get_date('U') > 0) { - $newitem->setDate((int)$item->get_date('U')); - } elseif ($extractor->getDate()) { - $newitem->setDate($extractor->getDate()); - } - - // add authors - if ($authors = $item->get_authors()) { - foreach ($authors as $author) { - // for some feeds, SimplePie stores author's name as email, e.g. http://feeds.feedburner.com/nymag/intel - if ($author->get_name() !== null) { - $newitem->addElement('dc:creator', $author->get_name()); - } elseif ($author->get_email() !== null) { - $newitem->addElement('dc:creator', $author->get_email()); + // Try to get first few paragraphs + if (isset($content_block) && ($content_block instanceof DOMElement)) { + $_paras = $content_block->getElementsByTagName('p'); + foreach ($_paras as $_para) { + $summary .= preg_replace("/[\n\r\t ]+/", ' ', $_para->textContent).' '; + if (strlen($summary) > 200) break; } + } else { + $summary = $html; } - } elseif ($authors = $extractor->getAuthors()) { - //TODO: make sure the list size is reasonable - foreach ($authors as $author) { - // TODO: xpath often selects authors from other articles linked from the page. - // for now choose first item - $newitem->addElement('dc:creator', $author); - break; + } + unset($_paras, $_para); + $summary = get_excerpt($summary); + $newitem->setDescription($summary); + if ($options->content) $newitem->setElement('content:encoded', $html); + } else { + if ($options->content) $newitem->setDescription($html); + } + + // set date + if ((int)$item->get_date('U') > 0) { + $newitem->setDate((int)$item->get_date('U')); + } elseif ($extractor->getDate()) { + $newitem->setDate($extractor->getDate()); + } + + // add authors + if ($authors = $item->get_authors()) { + foreach ($authors as $author) { + // for some feeds, SimplePie stores author's name as email, e.g. http://feeds.feedburner.com/nymag/intel + if ($author->get_name() !== null) { + $newitem->addElement('dc:creator', $author->get_name()); + } elseif ($author->get_email() !== null) { + $newitem->addElement('dc:creator', $author->get_email()); } } - - // add language - if ($detect_language) { - $language = $extractor->getLanguage(); - if (!$language) $language = $feed->get_language(); - if (($detect_language == 3 || (!$language && $detect_language == 2)) && $text_sample) { - try { - if ($use_cld) { - // Use PHP-CLD extension - $php_cld = 'CLD\detect'; // in quotes to prevent PHP 5.2 parse error - $res = $php_cld($text_sample); - if (is_array($res) && count($res) > 0) { - $language = $res[0]['code']; - } - } else { - //die('what'); - // Use PEAR's Text_LanguageDetect - if (!isset($l)) { - $l = new Text_LanguageDetect('libraries/language-detect/lang.dat', 'libraries/language-detect/unicode_blocks.dat'); - } - $l_result = $l->detect($text_sample, 1); - if (count($l_result) > 0) { - $language = $language_codes[key($l_result)]; - } + } elseif ($authors = $extractor->getAuthors()) { + //TODO: make sure the list size is reasonable + foreach ($authors as $author) { + // TODO: xpath often selects authors from other articles linked from the page. + // for now choose first item + $newitem->addElement('dc:creator', $author); + break; + } + } + + // add language + if ($detect_language) { + $language = $extractor->getLanguage(); + if (!$language) $language = $feed->get_language(); + if (($detect_language == 3 || (!$language && $detect_language == 2)) && $text_sample) { + try { + if ($use_cld) { + // Use PHP-CLD extension + $php_cld = 'CLD\detect'; // in quotes to prevent PHP 5.2 parse error + $res = $php_cld($text_sample); + if (is_array($res) && count($res) > 0) { + $language = $res[0]['code']; + } + } else { + //die('what'); + // Use PEAR's Text_LanguageDetect + if (!isset($l)) { + $l = new Text_LanguageDetect(); + $l->setNameMode(2); // return ISO 639-1 codes (e.g. "en") + } + $l_result = $l->detect($text_sample, 1); + if (count($l_result) > 0) { + $language = key($l_result); } - } catch (Exception $e) { - //die('error: '.$e); - // do nothing } - } - if ($language && (strlen($language) < 7)) { - $newitem->addElement('dc:language', $language); + } catch (Exception $e) { + //die('error: '.$e); + // do nothing } } - - // add MIME type (if it appeared in our exclusions lists) - if (isset($mime_info['mime'])) $newitem->addElement('dc:format', $mime_info['mime']); - // add effective URL (URL after redirects) - if (isset($effective_url)) { - //TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g. - //http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir--25th-March-2012-Special-Program-from-Liari-(Karachi) - //temporary measure: use utf8_encode() - $newitem->addElement('dc:identifier', remove_url_cruft(utf8_encode($effective_url))); - } else { - $newitem->addElement('dc:identifier', remove_url_cruft($item->get_permalink())); + if ($language && (strlen($language) < 7)) { + $newitem->addElement('dc:language', $language); } - - // add categories - if ($categories = $item->get_categories()) { - foreach ($categories as $category) { - if ($category->get_label() !== null) { - $newitem->addElement('category', $category->get_label()); - } + } + + // add MIME type (if it appeared in our exclusions lists) + if (isset($mime_info['mime'])) $newitem->addElement('dc:format', $mime_info['mime']); + // add effective URL (URL after redirects) + if (isset($effective_url)) { + //TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g. + //http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir-�-25th-March-2012-Special-Program-from-Liari-(Karachi) + //temporary measure: use utf8_encode() + $newitem->addElement('dc:identifier', remove_url_cruft(utf8_encode($effective_url))); + } else { + $newitem->addElement('dc:identifier', remove_url_cruft($item->get_permalink())); + } + + // add categories + if ($categories = $item->get_categories()) { + foreach ($categories as $category) { + if ($category->get_label() !== null) { + $newitem->addElement('category', $category->get_label()); } } - - // check for enclosures - if ($options->keep_enclosures) { - if ($enclosures = $item->get_enclosures()) { - foreach ($enclosures as $enclosure) { - // thumbnails - foreach ((array)$enclosure->get_thumbnails() as $thumbnail) { - $newitem->addElement('media:thumbnail', '', array('url'=>$thumbnail)); - } - if (!$enclosure->get_link()) continue; - $enc = array(); - // Media RSS spec ($enc): http://search.yahoo.com/mrss - // SimplePie methods ($enclosure): http://simplepie.org/wiki/reference/start#methods4 - $enc['url'] = $enclosure->get_link(); - if ($enclosure->get_length()) $enc['fileSize'] = $enclosure->get_length(); - if ($enclosure->get_type()) $enc['type'] = $enclosure->get_type(); - if ($enclosure->get_medium()) $enc['medium'] = $enclosure->get_medium(); - if ($enclosure->get_expression()) $enc['expression'] = $enclosure->get_expression(); - if ($enclosure->get_bitrate()) $enc['bitrate'] = $enclosure->get_bitrate(); - if ($enclosure->get_framerate()) $enc['framerate'] = $enclosure->get_framerate(); - if ($enclosure->get_sampling_rate()) $enc['samplingrate'] = $enclosure->get_sampling_rate(); - if ($enclosure->get_channels()) $enc['channels'] = $enclosure->get_channels(); - if ($enclosure->get_duration()) $enc['duration'] = $enclosure->get_duration(); - if ($enclosure->get_height()) $enc['height'] = $enclosure->get_height(); - if ($enclosure->get_width()) $enc['width'] = $enclosure->get_width(); - if ($enclosure->get_language()) $enc['lang'] = $enclosure->get_language(); - $newitem->addElement('media:content', '', $enc); + } + + // check for enclosures + if ($options->keep_enclosures) { + if ($enclosures = $item->get_enclosures()) { + foreach ($enclosures as $enclosure) { + // thumbnails + foreach ((array)$enclosure->get_thumbnails() as $thumbnail) { + $newitem->addElement('media:thumbnail', '', array('url'=>$thumbnail)); } + if (!$enclosure->get_link()) continue; + $enc = array(); + // Media RSS spec ($enc): http://search.yahoo.com/mrss + // SimplePie methods ($enclosure): http://simplepie.org/wiki/reference/start#methods4 + $enc['url'] = $enclosure->get_link(); + if ($enclosure->get_length()) $enc['fileSize'] = $enclosure->get_length(); + if ($enclosure->get_type()) $enc['type'] = $enclosure->get_type(); + if ($enclosure->get_medium()) $enc['medium'] = $enclosure->get_medium(); + if ($enclosure->get_expression()) $enc['expression'] = $enclosure->get_expression(); + if ($enclosure->get_bitrate()) $enc['bitrate'] = $enclosure->get_bitrate(); + if ($enclosure->get_framerate()) $enc['framerate'] = $enclosure->get_framerate(); + if ($enclosure->get_sampling_rate()) $enc['samplingrate'] = $enclosure->get_sampling_rate(); + if ($enclosure->get_channels()) $enc['channels'] = $enclosure->get_channels(); + if ($enclosure->get_duration()) $enc['duration'] = $enclosure->get_duration(); + if ($enclosure->get_height()) $enc['height'] = $enclosure->get_height(); + if ($enclosure->get_width()) $enc['width'] = $enclosure->get_width(); + if ($enclosure->get_language()) $enc['lang'] = $enclosure->get_language(); + $newitem->addElement('media:content', '', $enc); } } - /* } */ + } $output->addItem($newitem); unset($html); $item_count++; @@ -887,7 +892,7 @@ if (!$debug_mode) { } if ($add_to_cache) { ob_start(); - $output->genarateFeed(); + $output->genarateFeed(false); $output = ob_get_contents(); ob_end_clean(); if ($html_only && $item_count == 0) { @@ -898,299 +903,8 @@ if (!$debug_mode) { } echo $output; } else { - $output->genarateFeed(); + $output->genarateFeed(false); } if ($callback) echo ');'; } -/////////////////////////////// -// HELPER FUNCTIONS -/////////////////////////////// - -function url_allowed($url) { - global $options; - if (!empty($options->allowed_urls)) { - $allowed = false; - foreach ($options->allowed_urls as $allowurl) { - if (stristr($url, $allowurl) !== false) { - $allowed = true; - break; - } - } - if (!$allowed) return false; - } else { - foreach ($options->blocked_urls as $blockurl) { - if (stristr($url, $blockurl) !== false) { - return false; - } - } - } - return true; -} - -////////////////////////////////////////////// -// Convert $html to UTF8 -// (uses HTTP headers and HTML to find encoding) -// adapted from http://stackoverflow.com/questions/910793/php-detect-encoding-and-make-everything-utf-8 -////////////////////////////////////////////// -function convert_to_utf8($html, $header=null) -{ - $encoding = null; - if ($html || $header) { - if (is_array($header)) $header = implode("\n", $header); - if (!$header || !preg_match_all('/^Content-Type:\s+([^;]+)(?:;\s*charset=["\']?([^;"\'\n]*))?/im', $header, $match, PREG_SET_ORDER)) { - // error parsing the response - debug('Could not find Content-Type header in HTTP response'); - } else { - $match = end($match); // get last matched element (in case of redirects) - if (isset($match[2])) $encoding = trim($match[2], "\"' \r\n\0\x0B\t"); - } - // TODO: check to see if encoding is supported (can we convert it?) - // If it's not, result will be empty string. - // For now we'll check for invalid encoding types returned by some sites, e.g. 'none' - // Problem URL: http://facta.co.jp/blog/archives/20111026001026.html - if (!$encoding || $encoding == 'none') { - // search for encoding in HTML - only look at the first 50000 characters - // Why 50000? See, for example, http://www.lemonde.fr/festival-de-cannes/article/2012/05/23/deux-cretes-en-goguette-sur-la-croisette_1705732_766360.html - // TODO: improve this so it looks at smaller chunks first - $html_head = substr($html, 0, 50000); - if (preg_match('/^<\?xml\s+version=(?:"[^"]*"|\'[^\']*\')\s+encoding=("[^"]*"|\'[^\']*\')/s', $html_head, $match)) { - $encoding = trim($match[1], '"\''); - } elseif (preg_match('/
- $url = trim(str_replace('%20', ' ', $e->getAttribute($attr)));
- $url = str_replace(' ', '%20', $url);
- if (!preg_match('!https?://!i', $url)) {
- if ($absolute = SimplePie_IRI::absolutize($base, $url)) {
- $e->setAttribute($attr, $absolute);
- }
- }
- }
-}
-function makeAbsoluteStr($base, $url) {
- $base = new SimplePie_IRI($base);
- // remove '//' in URL path (causes URLs not to resolve properly)
- if (isset($base->path)) $base->path = preg_replace('!//+!', '/', $base->path);
- if (preg_match('!^https?://!i', $url)) {
- // already absolute
- return $url;
- } else {
- if ($absolute = SimplePie_IRI::absolutize($base, $url)) {
- return $absolute;
- }
- return false;
- }
-}
-// returns single page response, or false if not found
-function getSinglePage($item, $html, $url) {
- global $http, $extractor;
- debug('Looking for site config files to see if single page link exists');
- $site_config = $extractor->buildSiteConfig($url, $html);
- $splink = null;
- if (!empty($site_config->single_page_link)) {
- $splink = $site_config->single_page_link;
- } elseif (!empty($site_config->single_page_link_in_feed)) {
- // single page link xpath is targeted at feed
- $splink = $site_config->single_page_link_in_feed;
- // so let's replace HTML with feed item description
- $html = $item->get_description();
- }
- if (isset($splink)) {
- // Build DOM tree from HTML
- $readability = new Readability($html, $url);
- $xpath = new DOMXPath($readability->dom);
- // Loop through single_page_link xpath expressions
- $single_page_url = null;
- foreach ($splink as $pattern) {
- $elems = @$xpath->evaluate($pattern, $readability->dom);
- if (is_string($elems)) {
- $single_page_url = trim($elems);
- break;
- } elseif ($elems instanceof DOMNodeList && $elems->length > 0) {
- foreach ($elems as $item) {
- if ($item instanceof DOMElement && $item->hasAttribute('href')) {
- $single_page_url = $item->getAttribute('href');
- break 2;
- } elseif ($item instanceof DOMAttr && $item->value) {
- $single_page_url = $item->value;
- break 2;
- }
- }
- }
- }
- // If we've got URL, resolve against $url
- if (isset($single_page_url) && ($single_page_url = makeAbsoluteStr($url, $single_page_url))) {
- // check it's not what we have already!
- if ($single_page_url != $url) {
- // it's not, so let's try to fetch it...
- $_prev_ref = $http->referer;
- $http->referer = $single_page_url;
- if (($response = $http->get($single_page_url, true)) && $response['status_code'] < 300) {
- $http->referer = $_prev_ref;
- return $response;
- }
- $http->referer = $_prev_ref;
- }
- }
- }
- return false;
-}
-
-// based on content-type http header, decide what to do
-// param: HTTP headers string
-// return: array with keys: 'mime', 'type', 'subtype', 'action', 'name'
-// e.g. array('mime'=>'image/jpeg', 'type'=>'image', 'subtype'=>'jpeg', 'action'=>'link', 'name'=>'Image')
-function get_mime_action_info($headers) {
- global $options;
- // check if action defined for returned Content-Type
- $info = array();
- if (preg_match('!^Content-Type:\s*(([-\w]+)/([-\w\+]+))!im', $headers, $match)) {
- // look for full mime type (e.g. image/jpeg) or just type (e.g. image)
- // match[1] = full mime type, e.g. image/jpeg
- // match[2] = first part, e.g. image
- // match[3] = last part, e.g. jpeg
- $info['mime'] = strtolower(trim($match[1]));
- $info['type'] = strtolower(trim($match[2]));
- $info['subtype'] = strtolower(trim($match[3]));
- foreach (array($info['mime'], $info['type']) as $_mime) {
- if (isset($options->content_type_exc[$_mime])) {
- $info['action'] = $options->content_type_exc[$_mime]['action'];
- $info['name'] = $options->content_type_exc[$_mime]['name'];
- break;
- }
- }
- }
- return $info;
-}
-
-function remove_url_cruft($url) {
- // remove google analytics for the time being
- // regex adapted from http://navitronic.co.uk/2010/12/removing-google-analytics-cruft-from-urls/
- // https://gist.github.com/758177
- return preg_replace('/(\?|\&)utm_[a-z]+=[^\&]+/', '', $url);
-}
-
-function make_substitutions($string) {
- if ($string == '') return $string;
- global $item, $effective_url;
- $string = str_replace('{url}', htmlspecialchars($item->get_permalink()), $string);
- $string = str_replace('{effective-url}', htmlspecialchars($effective_url), $string);
- return $string;
-}
-
-function get_cache() {
- global $options, $valid_key;
- static $cache = null;
- if ($cache === null) {
- $frontendOptions = array(
- 'lifetime' => 10*60, // cache lifetime of 10 minutes
- 'automatic_serialization' => false,
- 'write_control' => false,
- 'automatic_cleaning_factor' => $options->cache_cleanup,
- 'ignore_user_abort' => false
- );
- $backendOptions = array(
- 'cache_dir' => ($valid_key) ? $options->cache_dir.'/rss-with-key/' : $options->cache_dir.'/rss/', // directory where to put the cache files
- 'file_locking' => false,
- 'read_control' => true,
- 'read_control_type' => 'strlen',
- 'hashed_directory_level' => $options->cache_directory_level,
- 'hashed_directory_perm' => 0777,
- 'cache_file_perm' => 0664,
- 'file_name_prefix' => 'ff'
- );
- // getting a Zend_Cache_Core object
- $cache = Zend_Cache::factory('Core', 'File', $frontendOptions, $backendOptions);
- }
- return $cache;
-}
-
-function debug($msg) {
- global $debug_mode;
- if ($debug_mode) {
- echo '* ',$msg,"\n";
- ob_flush();
- flush();
- }
-}
\ No newline at end of file