X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=inc%2F3rdparty%2Fmakefulltextfeed.php;h=a081f88b29f6800dab4cd2249366e35d45cd33dd;hb=90a1a78b1e2f4d40e1d9b8e6f46aca129a9d7bcf;hp=7104bc73b4b3b38a43028d46fa3096a1d540eaa4;hpb=d5501950e2470d52f6bf5954d2179010cdee0475;p=github%2Fwallabag%2Fwallabag.git
diff --git a/inc/3rdparty/makefulltextfeed.php b/inc/3rdparty/makefulltextfeed.php
old mode 100644
new mode 100755
index 7104bc73..a081f88b
--- a/inc/3rdparty/makefulltextfeed.php
+++ b/inc/3rdparty/makefulltextfeed.php
@@ -3,8 +3,8 @@
// Author: Keyvan Minoukadeh
// Copyright (c) 2013 Keyvan Minoukadeh
// License: AGPLv3
-// Version: 3.1
-// Date: 2013-03-05
+// Version: 3.2
+// Date: 2013-05-13
// More info: http://fivefilters.org/content-only/
// Help: http://help.fivefilters.org
@@ -25,14 +25,10 @@ along with this program. If not, see ";
+ } else {
+ $html = "Download {$mime_info['name']}";
+ }
+ $extracted_title = $mime_info['name'];
+ $do_content_extraction = false;
+ }
+ }
+ if ($do_content_extraction) {
+ $html = $single_page_response['body'];
+ // remove strange things
+ $html = str_replace('[>', '', $html);
+ $html = convert_to_utf8($html, $single_page_response['headers']);
+ debug("Retrieved single-page view from $effective_url");
+ }
unset($single_page_response);
}
+ }
+ if ($do_content_extraction) {
debug('--------');
debug('Attempting to extract content');
$extract_result = $extractor->process($html, $effective_url);
$readability = $extractor->readability;
- $content_block = ($extract_result) ? $extractor->getContent() : null;
+ $content_block = ($extract_result) ? $extractor->getContent() : null;
$extracted_title = ($extract_result) ? $extractor->getTitle() : '';
// Deal with multi-page articles
//die('Next: '.$extractor->getNextPageUrl());
$is_multi_page = (!$is_single_page && $extract_result && $extractor->getNextPageUrl());
- if ($options->multipage && $is_multi_page) {
+ if ($options->multipage && $is_multi_page && $options->content) {
debug('--------');
debug('Attempting to process multi-page article');
$multi_page_urls = array();
@@ -635,7 +610,7 @@ foreach ($items as $key => $item) {
// check it's not what we have already!
if (!in_array($next_page_url, $multi_page_urls)) {
// it's not, so let's attempt to fetch it
- $multi_page_urls[] = $next_page_url;
+ $multi_page_urls[] = $next_page_url;
$_prev_ref = $http->referer;
if (($response = $http->get($next_page_url, true)) && $response['status_code'] < 300) {
// make sure mime type is not something with a different action associated
@@ -660,13 +635,15 @@ foreach ($items as $key => $item) {
// did we successfully deal with this multi-page article?
if (empty($multi_page_content)) {
debug('Failed to extract all parts of multi-page article, so not going to include them');
- $multi_page_content[] = $readability->dom->createElement('p')->innerHTML = 'This article appears to continue on subsequent pages which we could not extract';
+ $_page = $readability->dom->createElement('p');
+ $_page->innerHTML = 'This article appears to continue on subsequent pages which we could not extract';
+ $multi_page_content[] = $_page;
}
foreach ($multi_page_content as $_page) {
$_page = $content_block->ownerDocument->importNode($_page, true);
$content_block->appendChild($_page);
}
- unset($multi_page_urls, $multi_page_content, $page_mime_info, $next_page_url);
+ unset($multi_page_urls, $multi_page_content, $page_mime_info, $next_page_url, $_page);
}
}
// use extracted title for both feed and item title if we're using single-item dummy feed
@@ -694,7 +671,11 @@ foreach ($items as $key => $item) {
$html .= $item->get_description();
} else {
$readability->clean($content_block, 'select');
- if ($options->rewrite_relative_urls) makeAbsolute($effective_url, $content_block);
+ // get base URL
+ $base_url = get_base_url($readability->dom);
+ if (!$base_url) $base_url = $effective_url;
+ // rewrite URLs
+ if ($options->rewrite_relative_urls) makeAbsolute($base_url, $content_block);
// footnotes
if (($links == 'footnotes') && (strpos($effective_url, 'wikipedia.org') === false)) {
$readability->addFootnotes($content_block);
@@ -713,7 +694,7 @@ foreach ($items as $key => $item) {
} else {
$html = $content_block->ownerDocument->saveXML($content_block); // essentially outerHTML
}
- unset($content_block);
+ //unset($content_block);
// post-processing cleanup
$html = preg_replace('!
[\s\h\v]*
!u', '', $html); if ($links == 'remove') { @@ -726,130 +707,155 @@ foreach ($items as $key => $item) { } } - if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment - $newitem->addElement('guid', 'http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()), array('isPermaLink'=>'false')); + if ($valid_key && isset($_GET['pubsub'])) { // used only on fivefilters.org at the moment + $newitem->addElement('guid', 'http://fivefilters.org/content-only/redirect.php?url='.urlencode($item->get_permalink()), array('isPermaLink'=>'false')); + } else { + $newitem->addElement('guid', $item->get_permalink(), array('isPermaLink'=>'true')); + } + // filter xss? + if ($xss_filter) { + debug('Filtering HTML to remove XSS'); + $html = htmLawed::hl($html, array('safe'=>1, 'deny_attribute'=>'style', 'comment'=>1, 'cdata'=>1)); + } + + // add content + if ($options->summary === true) { + // get summary + $summary = ''; + if (!$do_content_extraction) { + $summary = $html; } else { - $newitem->addElement('guid', $item->get_permalink(), array('isPermaLink'=>'true')); - } - // filter xss? - if ($xss_filter) { - debug('Filtering HTML to remove XSS'); - $html = htmLawed::hl($html, array('safe'=>1, 'deny_attribute'=>'style', 'comment'=>1, 'cdata'=>1)); - } - $newitem->setDescription($html); - - // set date - if ((int)$item->get_date('U') > 0) { - $newitem->setDate((int)$item->get_date('U')); - } elseif ($extractor->getDate()) { - $newitem->setDate($extractor->getDate()); - } - - // add authors - if ($authors = $item->get_authors()) { - foreach ($authors as $author) { - // for some feeds, SimplePie stores author's name as email, e.g. http://feeds.feedburner.com/nymag/intel - if ($author->get_name() !== null) { - $newitem->addElement('dc:creator', $author->get_name()); - } elseif ($author->get_email() !== null) { - $newitem->addElement('dc:creator', $author->get_email()); + // Try to get first few paragraphs + if (isset($content_block) && ($content_block instanceof DOMElement)) { + $_paras = $content_block->getElementsByTagName('p'); + foreach ($_paras as $_para) { + $summary .= preg_replace("/[\n\r\t ]+/", ' ', $_para->textContent).' '; + if (strlen($summary) > 200) break; } + } else { + $summary = $html; } - } elseif ($authors = $extractor->getAuthors()) { - //TODO: make sure the list size is reasonable - foreach ($authors as $author) { - // TODO: xpath often selects authors from other articles linked from the page. - // for now choose first item - $newitem->addElement('dc:creator', $author); - break; + } + unset($_paras, $_para); + $summary = get_excerpt($summary); + $newitem->setDescription($summary); + if ($options->content) $newitem->setElement('content:encoded', $html); + } else { + if ($options->content) $newitem->setDescription($html); + } + + // set date + if ((int)$item->get_date('U') > 0) { + $newitem->setDate((int)$item->get_date('U')); + } elseif ($extractor->getDate()) { + $newitem->setDate($extractor->getDate()); + } + + // add authors + if ($authors = $item->get_authors()) { + foreach ($authors as $author) { + // for some feeds, SimplePie stores author's name as email, e.g. http://feeds.feedburner.com/nymag/intel + if ($author->get_name() !== null) { + $newitem->addElement('dc:creator', $author->get_name()); + } elseif ($author->get_email() !== null) { + $newitem->addElement('dc:creator', $author->get_email()); } } - - // add language - if ($detect_language) { - $language = $extractor->getLanguage(); - if (!$language) $language = $feed->get_language(); - if (($detect_language == 3 || (!$language && $detect_language == 2)) && $text_sample) { - try { - if ($use_cld) { - // Use PHP-CLD extension - $php_cld = 'CLD\detect'; // in quotes to prevent PHP 5.2 parse error - $res = $php_cld($text_sample); - if (is_array($res) && count($res) > 0) { - $language = $res[0]['code']; - } - } else { - //die('what'); - // Use PEAR's Text_LanguageDetect - if (!isset($l)) { - $l = new Text_LanguageDetect('libraries/language-detect/lang.dat', 'libraries/language-detect/unicode_blocks.dat'); - } - $l_result = $l->detect($text_sample, 1); - if (count($l_result) > 0) { - $language = $language_codes[key($l_result)]; - } + } elseif ($authors = $extractor->getAuthors()) { + //TODO: make sure the list size is reasonable + foreach ($authors as $author) { + // TODO: xpath often selects authors from other articles linked from the page. + // for now choose first item + $newitem->addElement('dc:creator', $author); + break; + } + } + + // add language + if ($detect_language) { + $language = $extractor->getLanguage(); + if (!$language) $language = $feed->get_language(); + if (($detect_language == 3 || (!$language && $detect_language == 2)) && $text_sample) { + try { + if ($use_cld) { + // Use PHP-CLD extension + $php_cld = 'CLD\detect'; // in quotes to prevent PHP 5.2 parse error + $res = $php_cld($text_sample); + if (is_array($res) && count($res) > 0) { + $language = $res[0]['code']; + } + } else { + //die('what'); + // Use PEAR's Text_LanguageDetect + if (!isset($l)) { + $l = new Text_LanguageDetect(); + $l->setNameMode(2); // return ISO 639-1 codes (e.g. "en") + } + $l_result = $l->detect($text_sample, 1); + if (count($l_result) > 0) { + $language = key($l_result); } - } catch (Exception $e) { - //die('error: '.$e); - // do nothing } - } - if ($language && (strlen($language) < 7)) { - $newitem->addElement('dc:language', $language); + } catch (Exception $e) { + //die('error: '.$e); + // do nothing } } - - // add MIME type (if it appeared in our exclusions lists) - if (isset($mime_info['mime'])) $newitem->addElement('dc:format', $mime_info['mime']); - // add effective URL (URL after redirects) - if (isset($effective_url)) { - //TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g. - //http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir--25th-March-2012-Special-Program-from-Liari-(Karachi) - //temporary measure: use utf8_encode() - $newitem->addElement('dc:identifier', remove_url_cruft(utf8_encode($effective_url))); - } else { - $newitem->addElement('dc:identifier', remove_url_cruft($item->get_permalink())); + if ($language && (strlen($language) < 7)) { + $newitem->addElement('dc:language', $language); } - - // add categories - if ($categories = $item->get_categories()) { - foreach ($categories as $category) { - if ($category->get_label() !== null) { - $newitem->addElement('category', $category->get_label()); - } + } + + // add MIME type (if it appeared in our exclusions lists) + if (isset($mime_info['mime'])) $newitem->addElement('dc:format', $mime_info['mime']); + // add effective URL (URL after redirects) + if (isset($effective_url)) { + //TODO: ensure $effective_url is valid witout - sometimes it causes problems, e.g. + //http://www.siasat.pk/forum/showthread.php?108883-Pakistan-Chowk-by-Rana-Mubashir-�-25th-March-2012-Special-Program-from-Liari-(Karachi) + //temporary measure: use utf8_encode() + $newitem->addElement('dc:identifier', remove_url_cruft(utf8_encode($effective_url))); + } else { + $newitem->addElement('dc:identifier', remove_url_cruft($item->get_permalink())); + } + + // add categories + if ($categories = $item->get_categories()) { + foreach ($categories as $category) { + if ($category->get_label() !== null) { + $newitem->addElement('category', $category->get_label()); } } - - // check for enclosures - if ($options->keep_enclosures) { - if ($enclosures = $item->get_enclosures()) { - foreach ($enclosures as $enclosure) { - // thumbnails - foreach ((array)$enclosure->get_thumbnails() as $thumbnail) { - $newitem->addElement('media:thumbnail', '', array('url'=>$thumbnail)); - } - if (!$enclosure->get_link()) continue; - $enc = array(); - // Media RSS spec ($enc): http://search.yahoo.com/mrss - // SimplePie methods ($enclosure): http://simplepie.org/wiki/reference/start#methods4 - $enc['url'] = $enclosure->get_link(); - if ($enclosure->get_length()) $enc['fileSize'] = $enclosure->get_length(); - if ($enclosure->get_type()) $enc['type'] = $enclosure->get_type(); - if ($enclosure->get_medium()) $enc['medium'] = $enclosure->get_medium(); - if ($enclosure->get_expression()) $enc['expression'] = $enclosure->get_expression(); - if ($enclosure->get_bitrate()) $enc['bitrate'] = $enclosure->get_bitrate(); - if ($enclosure->get_framerate()) $enc['framerate'] = $enclosure->get_framerate(); - if ($enclosure->get_sampling_rate()) $enc['samplingrate'] = $enclosure->get_sampling_rate(); - if ($enclosure->get_channels()) $enc['channels'] = $enclosure->get_channels(); - if ($enclosure->get_duration()) $enc['duration'] = $enclosure->get_duration(); - if ($enclosure->get_height()) $enc['height'] = $enclosure->get_height(); - if ($enclosure->get_width()) $enc['width'] = $enclosure->get_width(); - if ($enclosure->get_language()) $enc['lang'] = $enclosure->get_language(); - $newitem->addElement('media:content', '', $enc); + } + + // check for enclosures + if ($options->keep_enclosures) { + if ($enclosures = $item->get_enclosures()) { + foreach ($enclosures as $enclosure) { + // thumbnails + foreach ((array)$enclosure->get_thumbnails() as $thumbnail) { + $newitem->addElement('media:thumbnail', '', array('url'=>$thumbnail)); } + if (!$enclosure->get_link()) continue; + $enc = array(); + // Media RSS spec ($enc): http://search.yahoo.com/mrss + // SimplePie methods ($enclosure): http://simplepie.org/wiki/reference/start#methods4 + $enc['url'] = $enclosure->get_link(); + if ($enclosure->get_length()) $enc['fileSize'] = $enclosure->get_length(); + if ($enclosure->get_type()) $enc['type'] = $enclosure->get_type(); + if ($enclosure->get_medium()) $enc['medium'] = $enclosure->get_medium(); + if ($enclosure->get_expression()) $enc['expression'] = $enclosure->get_expression(); + if ($enclosure->get_bitrate()) $enc['bitrate'] = $enclosure->get_bitrate(); + if ($enclosure->get_framerate()) $enc['framerate'] = $enclosure->get_framerate(); + if ($enclosure->get_sampling_rate()) $enc['samplingrate'] = $enclosure->get_sampling_rate(); + if ($enclosure->get_channels()) $enc['channels'] = $enclosure->get_channels(); + if ($enclosure->get_duration()) $enc['duration'] = $enclosure->get_duration(); + if ($enclosure->get_height()) $enc['height'] = $enclosure->get_height(); + if ($enclosure->get_width()) $enc['width'] = $enclosure->get_width(); + if ($enclosure->get_language()) $enc['lang'] = $enclosure->get_language(); + $newitem->addElement('media:content', '', $enc); } } - /* } */ + } $output->addItem($newitem); unset($html); $item_count++; @@ -886,7 +892,7 @@ if (!$debug_mode) { } if ($add_to_cache) { ob_start(); - $output->genarateFeed(); + $output->genarateFeed(false); $output = ob_get_contents(); ob_end_clean(); if ($html_only && $item_count == 0) { @@ -897,299 +903,8 @@ if (!$debug_mode) { } echo $output; } else { - $output->genarateFeed(); + $output->genarateFeed(false); } if ($callback) echo ');'; } -/////////////////////////////// -// HELPER FUNCTIONS -/////////////////////////////// - -function url_allowed($url) { - global $options; - if (!empty($options->allowed_urls)) { - $allowed = false; - foreach ($options->allowed_urls as $allowurl) { - if (stristr($url, $allowurl) !== false) { - $allowed = true; - break; - } - } - if (!$allowed) return false; - } else { - foreach ($options->blocked_urls as $blockurl) { - if (stristr($url, $blockurl) !== false) { - return false; - } - } - } - return true; -} - -////////////////////////////////////////////// -// Convert $html to UTF8 -// (uses HTTP headers and HTML to find encoding) -// adapted from http://stackoverflow.com/questions/910793/php-detect-encoding-and-make-everything-utf-8 -////////////////////////////////////////////// -function convert_to_utf8($html, $header=null) -{ - $encoding = null; - if ($html || $header) { - if (is_array($header)) $header = implode("\n", $header); - if (!$header || !preg_match_all('/^Content-Type:\s+([^;]+)(?:;\s*charset=["\']?([^;"\'\n]*))?/im', $header, $match, PREG_SET_ORDER)) { - // error parsing the response - debug('Could not find Content-Type header in HTTP response'); - } else { - $match = end($match); // get last matched element (in case of redirects) - if (isset($match[2])) $encoding = trim($match[2], "\"' \r\n\0\x0B\t"); - } - // TODO: check to see if encoding is supported (can we convert it?) - // If it's not, result will be empty string. - // For now we'll check for invalid encoding types returned by some sites, e.g. 'none' - // Problem URL: http://facta.co.jp/blog/archives/20111026001026.html - if (!$encoding || $encoding == 'none') { - // search for encoding in HTML - only look at the first 50000 characters - // Why 50000? See, for example, http://www.lemonde.fr/festival-de-cannes/article/2012/05/23/deux-cretes-en-goguette-sur-la-croisette_1705732_766360.html - // TODO: improve this so it looks at smaller chunks first - $html_head = substr($html, 0, 50000); - if (preg_match('/^<\?xml\s+version=(?:"[^"]*"|\'[^\']*\')\s+encoding=("[^"]*"|\'[^\']*\')/s', $html_head, $match)) { - $encoding = trim($match[1], '"\''); - } elseif (preg_match('/]+)/i', $html_head, $match)) { - $encoding = trim($match[1]); - } elseif (preg_match_all('/]+)>/i', $html_head, $match)) { - foreach ($match[1] as $_test) { - if (preg_match('/charset=["\']?([^"\']+)/i', $_test, $_m)) { - $encoding = trim($_m[1]); - break; - } - } - } - } - if (isset($encoding)) $encoding = trim($encoding); - // trim is important here! - if (!$encoding || (strtolower($encoding) == 'iso-8859-1')) { - // replace MS Word smart qutoes - $trans = array(); - $trans[chr(130)] = '‚'; // Single Low-9 Quotation Mark - $trans[chr(131)] = 'ƒ'; // Latin Small Letter F With Hook - $trans[chr(132)] = '„'; // Double Low-9 Quotation Mark - $trans[chr(133)] = '…'; // Horizontal Ellipsis - $trans[chr(134)] = '†'; // Dagger - $trans[chr(135)] = '‡'; // Double Dagger - $trans[chr(136)] = 'ˆ'; // Modifier Letter Circumflex Accent - $trans[chr(137)] = '‰'; // Per Mille Sign - $trans[chr(138)] = 'Š'; // Latin Capital Letter S With Caron - $trans[chr(139)] = '‹'; // Single Left-Pointing Angle Quotation Mark - $trans[chr(140)] = 'Œ'; // Latin Capital Ligature OE - $trans[chr(145)] = '‘'; // Left Single Quotation Mark - $trans[chr(146)] = '’'; // Right Single Quotation Mark - $trans[chr(147)] = '“'; // Left Double Quotation Mark - $trans[chr(148)] = '”'; // Right Double Quotation Mark - $trans[chr(149)] = '•'; // Bullet - $trans[chr(150)] = '–'; // En Dash - $trans[chr(151)] = '—'; // Em Dash - $trans[chr(152)] = '˜'; // Small Tilde - $trans[chr(153)] = '™'; // Trade Mark Sign - $trans[chr(154)] = 'š'; // Latin Small Letter S With Caron - $trans[chr(155)] = '›'; // Single Right-Pointing Angle Quotation Mark - $trans[chr(156)] = 'œ'; // Latin Small Ligature OE - $trans[chr(159)] = 'Ÿ'; // Latin Capital Letter Y With Diaeresis - $html = strtr($html, $trans); - } - if (!$encoding) { - debug('No character encoding found, so treating as UTF-8'); - $encoding = 'utf-8'; - } else { - debug('Character encoding: '.$encoding); - if (strtolower($encoding) != 'utf-8') { - debug('Converting to UTF-8'); - $html = SimplePie_Misc::change_encoding($html, $encoding, 'utf-8'); - /* - if (function_exists('iconv')) { - // iconv appears to handle certain character encodings better than mb_convert_encoding - $html = iconv($encoding, 'utf-8', $html); - } else { - $html = mb_convert_encoding($html, 'utf-8', $encoding); - } - */ - } - } - } - return $html; -} - -function makeAbsolute($base, $elem) { - $base = new SimplePie_IRI($base); - // remove '//' in URL path (used to prevent URLs from resolving properly) - // TODO: check if this is still the case - if (isset($base->path)) $base->path = preg_replace('!//+!', '/', $base->path); - foreach(array('a'=>'href', 'img'=>'src') as $tag => $attr) { - $elems = $elem->getElementsByTagName($tag); - for ($i = $elems->length-1; $i >= 0; $i--) { - $e = $elems->item($i); - //$e->parentNode->replaceChild($articleContent->ownerDocument->createTextNode($e->textContent), $e); - makeAbsoluteAttr($base, $e, $attr); - } - if (strtolower($elem->tagName) == $tag) makeAbsoluteAttr($base, $elem, $attr); - } -} -function makeAbsoluteAttr($base, $e, $attr) { - if ($e->hasAttribute($attr)) { - // Trim leading and trailing white space. I don't really like this but - // unfortunately it does appear on some sites. e.g.