From fb9df0c269f36703909b8b259abbdbed29881ecd Mon Sep 17 00:00:00 2001
From: tcit ".$data." Para 1 Para 2 Para 1 Para 2 The story begins...".$this->settings["title"]."
";
+
+ list($text, $entries) = $this->generateText();
+
+ if($this->settings["toc"]) {
+ $toc = $this->generateTOC($entries); //Generate TOC to get the right length
+ $toc = $this->generateTOC($entries, strlen($prefix)+strlen($toc)+strlen($title)); //Generate the real TOC
+ }
+
+ $suffix = "";
+
+ return $prefix.$toc.$title.$text.$suffix;
+ }
+
+ /**
+ * Generate the body's text and the chapter entries
+ * @return array($string, $entries) $string is the html data, $entries
+ * contains the level, the title and the position of the titles.
+ */
+ public function generateText(){
+ $str = "";
+ $entries = array();
+
+ for($i = 0; $i < sizeof($this->parts); $i++){
+ list($type, $data) = $this->parts[$i];
+ $id = "title_".$i;
+ switch($type){
+ case self::PARAGRAPH:
+ $str .= "".$data."
";
+ break;
+ case self::H3:
+ $entries[] = array("level" => 3, "position" => strlen($str), "title" => $data, "id" => $id);
+ $str .= "".$data."
";
+ break;
+ case self::IMAGE:
+ $str .= "";
+ break;
+ }
+ }
+ return array($str, $entries);
+ }
+
+ /**
+ * Generate a TOC
+ * @param $entries The entries array generated by generateText
+ * @param $base The zero position
+ */
+ public function generateTOC($entries, $base = 0){
+ $toc = "
Contents
";
+ $toc .= " ";
+ }
+ return $toc."".$entry["title"]." Contents
";
+ $toc .= "
";
+
+ return $toc;
+ }
+ /**
+ * Get the images (an array containing the jpeg data). Array entry 0 will
+ * correspond to image record 0.
+ * @return array
+ */
+ public function getImages(){
+ return $this->images;
+ }
+
+ /**
+ * Get the metadata in the form of a hashtable (for example, title or author).
+ * @return array
+ */
+ public function getMetaData(){
+ return $this->metadata;
+ }
+
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/OnlineArticle.php b/inc/3rdparty/libraries/MOBIClass/OnlineArticle.php
new file mode 100644
index 00000000..ec3182fe
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/OnlineArticle.php
@@ -0,0 +1,116 @@
+init();
+ if(!isset($this->metadata["title"])){
+ $this->metadata["title"] = CharacterEntities::convert(strip_tags($r->getTitle()->innerHTML));
+ }
+ if(!isset($this->metadata["author"])){
+ $parts = parse_url($url);
+ $this->metadata["author"] = $parts["host"];
+ }
+
+ $article = $r->getContent()->innerHTML;
+ if(substr($article, 0, 5) == "".$article."";
+ }else{
+ $article = "".$article."";
+ }
+ $doc = new DOMDocument();
+ @$doc->loadHTML($article) or die($article);
+ $doc->normalizeDocument();
+
+ $this->images = $this->handleImages($doc, $url);
+ $this->text = $doc->saveHTML();
+ }
+
+ /**
+ * Get the text data to be integrated in the MOBI file
+ * @return string
+ */
+ public function getTextData(){
+ return $this->text;
+ }
+ /**
+ * Get the images (an array containing the jpeg data). Array entry 0 will
+ * correspond to image record 0.
+ * @return array
+ */
+ public function getImages(){
+ return $this->images;
+ }
+ /**
+ * Get the metadata in the form of a hashtable (for example, title or author).
+ * @return array
+ */
+ public function getMetaData(){
+ return $this->metadata;
+ }
+ /**
+ *
+ * @param DOMElement $dom
+ * @return array
+ */
+ private function handleImages($dom, $url){
+ $images = array();
+
+ $parts = parse_url($url);
+
+ $savedImages = array();
+
+ $imgElements = $dom->getElementsByTagName('img');
+ foreach($imgElements as $img) {
+ $src = $img->getAttribute("src");
+
+ $is_root = false;
+ if(substr($src, 0, 1) == "/"){
+ $is_root = true;
+ }
+
+ $parsed = parse_url($src);
+
+ if(!isset($parsed["host"])){
+ if($is_root){
+ $src = http_build_url($url, $parsed, HTTP_URL_REPLACE);
+ }else{
+ $src = http_build_url($url, $parsed, HTTP_URL_JOIN_PATH);
+ }
+ }
+ $img->setAttribute("src", "");
+ if(isset($savedImages[$src])){
+ $img->setAttribute("recindex", $savedImages[$src]);
+ }else{
+ $image = ImageHandler::DownloadImage($src);
+
+ if($image !== false){
+ $images[$this->imgCounter] = new FileRecord(new Record($image));
+
+ $img->setAttribute("recindex", $this->imgCounter);
+ $savedImages[$src] = $this->imgCounter;
+ $this->imgCounter++;
+ }
+ }
+ }
+
+ return $images;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/PalmRecord.php b/inc/3rdparty/libraries/MOBIClass/PalmRecord.php
new file mode 100644
index 00000000..d0de8dfe
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/PalmRecord.php
@@ -0,0 +1,136 @@
+elements = new FileElement(array(
+ "compression"=>new FileShort(),
+ "unused"=>new FileShort(),
+ "textLength"=>new FileInt(),
+ "recordCount"=>new FileShort(),
+ "recordSize"=>new FileShort(),
+ "encryptionType"=>new FileShort(),
+ "unused2"=>new FileShort(),
+ //MOBI Header
+ "mobiIdentifier"=>new FileString("MOBI", 4),
+ "mobiHeaderLength"=>new FileInt(),
+ "mobiType"=>new FileInt(),
+ "textEncoding"=>new FileInt(),
+ "uniqueID"=>new FileInt(),
+ "fileVersion"=>new FileInt(),
+ "reserved"=>new FileString(40),
+ "firstNonBookIndex"=>new FileInt(),
+ "fullNameOffset"=>new FileInt(),
+ "fullNameLength"=>new FileInt(),
+ "locale"=>new FileInt(),
+ "inputLanguage"=>new FileInt(),
+ "outputLanguage"=>new FileInt(),
+ "minimumVersion"=>new FileInt(),
+ "firstImageIndex"=>new FileInt(),
+ "huffmanRecordOffset"=>new FileInt(),
+ "huffmanRecordCount"=>new FileInt(),
+ "unused3"=>new FileString(8),
+ "exthFlags"=>new FileInt(0x40),
+ "unknown"=>new FileString(32),
+ "drmOffset"=>new FileInt(0xFFFFFFFF),
+ "drmCount"=>new FileShort(0xFFFFFFFF),
+ "drmSize"=>new FileShort(),
+ "drmFlags"=>new FileInt(),
+ "mobiFiller"=>new FileString(72),
+ //EXTH Header
+ "exthIdentifier"=>new FileString("EXTH", 4),
+ "exthHeaderLength"=>new FileInt(),
+ "exthRecordCount"=>new FileInt(),
+ "exthRecords"=>new FileElement(),
+ "exthPadding"=>new FileString(),
+ //"fullNamePadding"=>new FileString(100),
+ "fullName"=>new FileString()
+ ));
+
+ //Set values from the info block
+ foreach($settings->values as $name => $val){
+ //echo $name.", ";
+ if($this->elements->exists($name)){
+ $this->elements->get($name)->set($settings->get($name));
+ }
+ }
+
+ $els = $settings->values;
+
+ $exthElems = new FileElement();
+ $i = 0;
+ $l = 0;
+ foreach($els as $name=>$val){
+ $type = EXTHHelper::textToType($name);
+ if($type !== false){
+ $type = new FileInt($type);
+ $length = new FileInt(8+strlen($val));
+ $data = new FileString($val);
+ $l += 8+strlen($val);
+ $exthElems->add("type".$i, $type);
+ $exthElems->add("length".$i, $length);
+ $exthElems->add("data".$i, $data);
+ $i++;
+ }
+ }
+
+ if($images > 0){
+ $this->elements->get("firstImageIndex")->set($textRecords+1);
+ }
+ $this->elements->get("firstNonBookIndex")->set($textRecords+2+$images);
+ $this->elements->get("reserved")->set(str_pad("", 40, chr(255), STR_PAD_RIGHT));
+ $this->elements->get("exthRecordCount")->set($i);
+ $this->elements->set("exthRecords", $exthElems);
+ $pad = $l%4;
+ $pad = (4-$pad)%4;
+ $this->elements->get("exthPadding")->set(str_pad("", $pad, "\0", STR_PAD_RIGHT));
+ $this->elements->get("exthHeaderLength")->set(12+$l+$pad);
+
+
+ $this->elements->get("recordCount")->set($textRecords);
+
+ $this->elements->get("fullNameOffset")->set($this->elements->offsetToEntry("fullName"));
+ $this->elements->get("fullNameLength")->set(strlen($settings->get("title")));
+ $this->elements->get("fullName")->set($settings->get("title"));
+ $this->elements->get("textLength")->set($textLength);
+ }
+
+ public function getByteLength(){
+ return $this->getLength();
+ }
+
+ public function getLength(){
+ return $this->elements->getByteLength();
+ }
+
+ public function get(){
+ return $this;
+ }
+
+ public function set($elements){
+ throw new Exception("Unallowed set");
+ }
+
+ public function serialize() {
+ return $this->elements->serialize();
+ }
+
+ public function unserialize($data) {
+ $this->elements->unserialize($data);
+ }
+
+ public function __toString(){
+ $output = "PalmDoc Record (".$this->getByteLength()." bytes):\n";
+ $output .= $this->elements;
+ return $output;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/Prc.php b/inc/3rdparty/libraries/MOBIClass/Prc.php
new file mode 100644
index 00000000..c40b5805
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/Prc.php
@@ -0,0 +1,97 @@
+new FileString(32),
+ "attributes"=>new FileShort(),
+ "version"=>new FileShort(),
+ "creationTime"=>new FileDate(),
+ "modificationTime"=>new FileDate(),
+ "backupTime"=>new FileDate(),
+ "modificationNumber"=>new FileInt(),
+ "appInfoID"=>new FileInt(),
+ "sortInfoID"=>new FileInt(),
+ "prcType"=>new FileString(4),
+ "creator"=>new FileString(4),
+ "uniqueIDSeed"=>new FileInt(),
+ "nextRecordListID"=>new FileInt(),
+ "numberRecords"=>new FileShort(),
+ "recordList"=>new FileElement(),
+ "filler"=>new FileShort(),
+ "records"=>new FileElement()
+ ));
+
+ //Set values from the info block
+ foreach($this->elements as $name => $val){
+ if($settings->exists($name)){
+ $this->get($name)->set($settings->get($name));
+ }
+ }
+
+ $this->get("numberRecords")->set(sizeof($records));
+
+ $i = 0;
+ foreach($records as $record){
+ $offset = new FileInt();
+ $attr = new FileByte();
+ $uniqueID = new FileTri($i);
+
+ $this->elements["recordList"]->add("Rec".$i, new FileElement(array(
+ "offset"=>$offset,
+ "attribute"=>$attr,
+ "uniqueID"=>$uniqueID
+ )));
+
+ $this->elements["records"]->add("Rec".$i, $record);
+ $i++;
+ }
+
+ $this->updateOffsets($records);
+ }
+
+ public function getByteLength(){
+ throw new Exception("Test");
+ }
+
+ public function updateOffsets($records){
+ $base = $this->offsetToEntry("records");
+
+ $i = 0;
+
+ foreach($records as $record){
+ $el = $this->elements["recordList"]->get("Rec".$i);
+
+ $local = $this->elements["records"]->offsetToEntry("Rec".$i);
+
+ $el->get("offset")->set($base+$local);
+
+ $i++;
+ }
+ }
+
+ public function save($file){
+ $handle = fopen($file, "w");
+ fwrite($handle, $this->serialize());
+ fclose($handle);
+ }
+
+ public function output(){
+ echo $this->serialize();
+ }
+
+ public function __toString(){
+ $output = "Prc (".$this->getByteLength()." bytes): {\n";
+ foreach($this->elements as $key=>$value){
+ $output .= "\t".$key.": ".$value."\n";
+ }
+ $output .= "}";
+ return $output;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/PreprocessedArticle.php b/inc/3rdparty/libraries/MOBIClass/PreprocessedArticle.php
new file mode 100644
index 00000000..2e992404
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/PreprocessedArticle.php
@@ -0,0 +1,89 @@
+text = $textData;
+ $this->metadata = $metadata;
+
+ $this->images = $this->downloadImages($imageLinks);
+ }
+
+ /**
+ * Create a Preprocessed article from a json string
+ * @param string $json JSON data. Should be of the following format:
+ * {"text": "TEXT", "images: ["imageURL1", "imageURL2"], "metadata": {"key": "value"}}
+ *
+ * Note: Any image tags should have the recindex attribute set to the appropriate index (the
+ * same index as the image in the array)
+ * @return PreprocessedArticle The generated preprocessed array
+ */
+ static public function CreateFromJson($json){
+ $data = json_decode($json);
+ return new PreprocessedArticle($data["text"], $data["images"], $data["metadata"]);
+ }
+
+ /**
+ * Get the text data to be integrated in the MOBI file
+ * @return string
+ */
+ public function getTextData(){
+ return $this->text;
+ }
+ /**
+ * Get the images (an array containing the jpeg data). Array entry 0 will
+ * correspond to image record 0.
+ * @return array
+ */
+ public function getImages(){
+ return $this->images;
+ }
+ /**
+ * Get the metadata in the form of a hashtable (for example, title or author).
+ * @return array
+ */
+ public function getMetaData(){
+ return $this->metadata;
+ }
+ /**
+ *
+ * @param DOMElement $dom
+ * @return array
+ */
+ private function downloadImages($links){
+ $images = array();
+ foreach($links as $link) {
+ $imgFile = @imagecreatefromstring(Http::Request($link));
+
+ if($imgFile === false){
+ $imgFile = @imagecreate(1, 1);
+ $black = @imagecolorallocate($imgFile, 255, 255, 255);
+ }
+ if($imgFile !== false){
+ @imagefilter($imgFile, IMG_FILTER_GRAYSCALE);
+
+ ob_start();
+ @imagejpeg($imgFile);
+ $image = ob_get_contents();
+ ob_end_clean();
+
+ $images[$this->imgCounter] = new FileRecord(new Record($image));
+ imagedestroy($imgFile);
+
+ $this->imgCounter++;
+ }
+ }
+
+ return $images;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/RecognizeURL.php b/inc/3rdparty/libraries/MOBIClass/RecognizeURL.php
new file mode 100644
index 00000000..6319ed57
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/RecognizeURL.php
@@ -0,0 +1,16 @@
+
diff --git a/inc/3rdparty/libraries/MOBIClass/Record.php b/inc/3rdparty/libraries/MOBIClass/Record.php
new file mode 100644
index 00000000..3cb39582
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/Record.php
@@ -0,0 +1,96 @@
+data = $data;
+ if($length >= 0){
+ $this->length = $length;
+ }else{
+ $this->length = strlen($data);
+ }
+ }
+
+ public function compress($compression_method){
+ switch($compression_method){
+ case NO_COMPRESSION:
+ //Finished!
+ break;
+ case PALMDOC_COMPRESSION:
+ throw new Exception("Not implemented yet");
+ break;
+ case HUFF:
+ throw new Exception("Not implemented yet");
+ break;
+ default:
+ throw new Exception("Invalid argument");
+ }
+ }
+
+ public function getByteLength(){
+ return $this->getLength();
+ }
+
+ /**
+ * Get the length of the record
+ * @return int Length of the data
+ */
+ public function getLength(){
+ return $this->length;
+ }
+
+ /**
+ * Get the data contained in the record
+ * @return string Data contained in the record
+ */
+ public function get(){
+ return $this->data;
+ }
+
+ /**
+ * Set the data contained in the record
+ * @param string $value Data contained in the record
+ */
+ public function set($value){
+ $this->data = $value;
+ }
+
+ public function serialize(){
+ return $this->data;
+ }
+ public function unserialize($data){
+ __construct($data);
+ }
+
+ public function __toString() {
+ $toShow = $this->data;
+ if(strlen($this->data) > 103){
+ $toShow = substr($this->data, 0, 100)."...";
+ }
+ $out = "Record: {\n";
+ $out .= "\t".htmlspecialchars($toShow)."\n";
+ $out .= "}";
+ return $out;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/RecordFactory.php b/inc/3rdparty/libraries/MOBIClass/RecordFactory.php
new file mode 100644
index 00000000..12806fe3
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/RecordFactory.php
@@ -0,0 +1,115 @@
+settings = $settings;
+ }
+
+ /**
+ * Create records from a data string
+ * @param string $data
+ * @return array(Record)
+ */
+ public function createRecords($data){
+ $records = array();
+ $size = $this->settings->get("recordSize");
+ $compression = $this->settings->get("compression");
+
+ $dataEntries = mb_str_split($data, $size);
+
+ for($i = 0, $len = sizeof($dataEntries); $i < $len; $i++){
+ $records[$i] = new Record($dataEntries[$i]);
+ $records[$i]->compress($compression);
+ }
+
+ return $records;
+ }
+
+ public function createEOFRecord(){
+ return new Record(0xe98e0d0a);
+ }
+
+ public function createFCISRecord($textLength){
+ $r = "FCIS";
+ $r .= $this->asString(20, 4);
+ $r .= $this->asString(16, 4);
+ $r .= $this->asString(1, 4);
+ $r .= $this->asString(0, 4);
+ $r .= $this->asString($textLength, 4);
+ $r .= $this->asString(0, 4);
+ $r .= $this->asString(32, 4);
+ $r .= $this->asString(8, 4);
+ $r .= $this->asString(1, 2);
+ $r .= $this->asString(1, 2);
+ $r .= $this->asString(0, 4);
+ return new Record($r);
+ }
+
+ public function createFLISRecord(){
+ $r = "FLIS";
+ $r .= $this->asString(8, 4);
+ $r .= $this->asString(65, 2);
+ $r .= $this->asString(0, 2);
+ $r .= $this->asString(0, 4);
+ $r .= $this->asString(-1, 4);
+ $r .= $this->asString(1, 2);
+ $r .= $this->asString(3, 2);
+ $r .= $this->asString(3, 4);
+ $r .= $this->asString(1, 4);
+ $r .= $this->asString(-1, 4);
+ return new Record($r);
+ }
+
+ private function asString($int, $size){
+ $out = "";
+ for($i = 0; $i < $size; $i++){
+ if($i > 0) $out = " ".$out;
+ $byte = dechex($int & 0xFF);
+ if(strlen($byte) == 1) $byte = "0".$byte;
+ $out = $byte.$out;
+ $int = $int >> 8;
+ }
+ return $out;
+ }
+
+ public function __toString() {
+ $out = "Record Factory: {\n";
+ $out .= "\tRecord Size: ".$this->settings->get("recordSize")."\n";
+ $out .= "\tCompression: ".$this->settings->get("compression")."\n";
+ $out .= "}";
+ return $out;
+ }
+}
+function mb_str_split($string, $split_length = 1){
+ mb_internal_encoding('UTF-8');
+ mb_regex_encoding('UTF-8');
+
+ $split_length = ($split_length <= 0) ? 1 : $split_length;
+
+ $mb_strlen = mb_strlen($string, 'utf-8');
+
+ $array = array();
+
+ for($i = 0; $i < $mb_strlen; $i += $split_length){
+ $array[] = mb_substr($string, $i, $split_length);
+ }
+
+ return $array;
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/Settings.php b/inc/3rdparty/libraries/MOBIClass/Settings.php
new file mode 100644
index 00000000..ddcf2054
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/Settings.php
@@ -0,0 +1,97 @@
+values = array(
+ "attributes"=>0,
+ "version"=>0,
+ "creationTime"=>time()+94694400,
+ "modificationTime"=>time()+94694400,
+ "backupTime"=>0,
+ "modificationNumber"=>0,
+ "appInfoID"=>0,
+ "sortInfoID"=>0,
+ "prcType"=>"BOOK",
+ "creator"=>"MOBI",
+ "uniqueIDSeed"=>rand(),
+ "nextRecordListID"=>0,
+ "recordAttributes"=>0,
+ "compression"=>NO_COMPRESSION,
+ "recordSize"=>RECORD_SIZE,
+ "encryptionType"=>NO_ENCRYPTION,
+ "mobiIdentifier"=>"MOBI",
+ "mobiHeaderLength"=>0xe8,
+ "mobiType"=>MOBIPOCKET_BOOK,
+ "textEncoding"=>UTF8,
+ "uniqueID"=>rand(),
+ "fileVersion"=>6,
+ "locale"=>0x09,
+ "inputLanguage"=>0,
+ "outputLanguage"=>0,
+ "minimumVersion"=>6,
+ "huffmanRecordOffset"=>0,
+ "huffmanRecordCount"=>0,
+ "exthFlags"=>0x40,
+ "drmOffset"=>0xFFFFFFFF,
+ "drmCount"=>0,
+ "drmSize"=>0,
+ "drmFlags"=>0,
+ "extraDataFlags"=>0,
+ "exthIdentifier"=>"EXTH",
+ // These can be changed without any risk
+ "title"=>"Unknown title",
+ "author"=>"Unknown author",
+ "subject"=>"Unknown subject"
+ );
+
+ foreach($additionalSettings as $key=>$value){
+ $this->values[$key] = $value;
+ }
+ }
+
+ /**
+ * Get a value from the settings
+ * @param string $key Key of the setting
+ * @return mixed The value of the setting
+ */
+ public function get($key){
+ return $this->values[$key];
+ }
+
+ /**
+ * Checks if a value is set
+ * @param string $key Key of the setting
+ * @return bool True if the value exists
+ */
+ public function exists($key){
+ return isset($this->values[$key]);
+ }
+
+ public function __toString() {
+ $out = "Settings: {\n";
+ foreach($this->values as $key=>$value){
+ $out .= "\t".$key.": ".$value."\n";
+ }
+ $out .= "}";
+ return $out;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/constants.php b/inc/3rdparty/libraries/MOBIClass/constants.php
new file mode 100644
index 00000000..bd363118
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/constants.php
@@ -0,0 +1,11 @@
+id = intval(substr($ending, 0, strpos($ending, "/")));
+
+ for($i = 1; $i <= max(1, $this->chapterCount); $i++){
+ $this->addChapter($i);
+ }
+ }
+
+ private function addChapter($n){
+ $doc = new DOMDocument();
+ $file = Http::Request(self::$prefix.$this->id."/".$n."/");
+ @$doc->loadHTML($file) or die($file);
+
+ if(!$this->downloadedMetadata){
+ $this->loadMetadata($doc);
+ $this->downloadedMetadata = true;
+ }
+ if($this->chapterCount < 0){
+ $this->chapterCount = $this->getNumberChapters($doc);
+
+ if($this->chapterCount > 4){
+ die("Too many files to download, don't use php for this!");
+ }
+ }
+
+ $textEl = $doc->getElementById("storytext");
+ if($textEl == null) die("Error: ".$doc->saveHTML());
+ $horizontalRulebars = $doc->getElementsByTagName('hr');
+ /**
+ * @var DOMNode
+ */
+ $hr;
+ foreach($horizontalRulebars as $hr) {
+ $hr->setAttribute("size", null);
+ $hr->setAttribute("noshade", null);
+ }
+ $text = $this->innerHtml($textEl);
+
+ $title = "";
+ $selects = $doc->getElementsByTagName('select');
+ foreach($selects as $select) {
+ if($select->hasAttribute("name") && $select->getAttribute("name") == "chapter"){
+ $options = $select->getElementsByTagName("option");
+
+ $test = $n.". ";
+ foreach($options as $option){
+ $val = $option->nodeValue;
+ if(substr($val, 0, strlen($test)) == $test){
+ $title = substr($val, strlen($test));
+ break;
+ }
+ }
+ break;
+ }
+ }
+ $this->addPage($text, $title);
+ }
+
+ private function getNumberChapters($doc){
+ $selects = $doc->getElementsByTagName('select');
+ foreach($selects as $select) {
+ if($select->hasAttribute("name") && $select->getAttribute("name") == "chapter"){
+ $options = $select->getElementsByTagName("option");
+
+ $count = $options->length;
+ return $count;
+ }
+ }
+ }
+
+ private function loadMetadata($doc){
+ //Author
+ $links = $doc->getElementsByTagName('a');
+ foreach($links as $link) {
+ if($link == null){
+ var_dump($link);
+ }
+ if($link->hasAttribute("href") && substr($link->getAttribute("href"), 0, 3) == "/u/"){
+ $this->setMetadata("author", $link->nodeValue);
+ }
+ }
+ //Title
+ /*
+ $links = $doc->getElementsByTagName('link');
+ foreach($links as $link) {
+ if($link->hasAttribute("rel") && $link->getAttribute("rel") == "canonical"){
+ $url = $link->getAttribute("href");
+ $title = str_replace("_", " ", substr($url, strrpos($url, "/")+1));
+ $this->setMetadata("title", $title);
+ }
+ }*/
+
+ //TODO: Find a more reliable way to extract the title
+ $title = $doc->getElementsByTagName("b")->item(0)->nodeValue;
+ $this->setMetadata("title", $title);
+ }
+
+ private function innerHtml($node){
+ $doc = new DOMDocument();
+ foreach ($node->childNodes as $child)
+ $doc->appendChild($doc->importNode($child, true));
+
+ return $doc->saveHTML();
+ }
+
+ public static function Matches($url){
+ //TODO: Implement with regex
+ return strpos($url, self::$prefix) !== false;
+ }
+}
+?>
diff --git a/inc/3rdparty/libraries/MOBIClass/http_build_url.php b/inc/3rdparty/libraries/MOBIClass/http_build_url.php
new file mode 100644
index 00000000..b475edb0
--- /dev/null
+++ b/inc/3rdparty/libraries/MOBIClass/http_build_url.php
@@ -0,0 +1,94 @@
+registerNodeClass('DOMElement', 'JSLikeHTMLElement');
+* $doc->loadHTML(' ";
+ }
+ $toc .= "".($i+1).". ".$entry["title"]." Chapter 2
]*>[ \n\r\t]*){2,}/i',
+ 'replaceFonts' => '/<(\/?)font[^>]*>/i',
+ // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
+ 'normalize' => '/\s{2,}/',
+ 'killBreaks' => '/(
(\s| ?)*){1,}/',
+ 'video' => '/http:\/\/(www\.)?(youtube|vimeo)\.com/i',
+ 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
+ );
+
+ /* constants */
+ const FLAG_STRIP_UNLIKELYS = 1;
+ const FLAG_WEIGHT_CLASSES = 2;
+ const FLAG_CLEAN_CONDITIONALLY = 4;
+
+ /**
+ * Create instance of Readability
+ * @param string UTF-8 encoded string
+ * @param string (optional) URL associated with HTML (used for footnotes)
+ */
+ function __construct($html, $url=null)
+ {
+ /* Turn all double br's into p's */
+ /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
+ $html = preg_replace($this->regexps['replaceBrs'], '
', $html); + $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); + $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); + $this->dom = new DOMDocument(); + $this->dom->preserveWhiteSpace = false; + $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); + @$this->dom->loadHTML($html); + $this->url = $url; + } + + /** + * Get article title element + * @return DOMElement + */ + public function getTitle() { + return $this->articleTitle; + } + + /** + * Get article content element + * @return DOMElement + */ + public function getContent() { + return $this->articleContent; + } + + /** + * Runs readability. + * + * Workflow: + * 1. Prep the document by removing script tags, css, etc. + * 2. Build readability's DOM tree. + * 3. Grab the article content from the current dom tree. + * 4. Replace the current DOM tree with the new one. + * 5. Read peacefully. + * + * @return boolean true if we found content, false otherwise + **/ + public function init() + { + $this->removeScripts($this->dom); + + // Assume successful outcome + $this->success = true; + + $bodyElems = $this->dom->getElementsByTagName('body'); + if ($bodyElems->length > 0) { + if ($this->bodyCache == null) { + $this->bodyCache = $bodyElems->item(0)->innerHTML; + } + if ($this->body == null) { + $this->body = $bodyElems->item(0); + } + } + + $this->prepDocument(); + + //die($this->dom->documentElement->parentNode->nodeType); + //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); + //die($this->getInnerHTML($this->dom->documentElement)); + + /* Build readability's DOM tree */ + $overlay = $this->dom->createElement('div'); + $innerDiv = $this->dom->createElement('div'); + $articleTitle = $this->getArticleTitle(); + $articleContent = $this->grabArticle(); + + if (!$articleContent) { + $this->success = false; + $articleContent = $this->dom->createElement('div'); + $articleContent->setAttribute('id', 'readability-content'); + $articleContent->innerHTML = '
Sorry, Readability was unable to parse this page for content.
'; + } + + $overlay->setAttribute('id', 'readOverlay'); + $innerDiv->setAttribute('id', 'readInner'); + + /* Glue the structure of our document together. */ + $innerDiv->appendChild($articleTitle); + $innerDiv->appendChild($articleContent); + $overlay->appendChild($innerDiv); + + /* Clear the old HTML, insert the new content. */ + $this->body->innerHTML = ''; + $this->body->appendChild($overlay); + //document.body.insertBefore(overlay, document.body.firstChild); + $this->body->removeAttribute('style'); + + $this->postProcessContent($articleContent); + + // Set title and content instance variables + $this->articleTitle = $articleTitle; + $this->articleContent = $articleContent; + + return $this->success; + } + + /** + * Debug + */ + protected function dbg($msg) { + if ($this->debug) echo '* ',$msg, '').replace(readability.regexps.replaceFonts, '<$1span>'); + // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. + // Manipulating innerHTML as it's done in JS is not possible in PHP. + } + + /** + * For easier reading, convert this document to have footnotes at the bottom rather than inline links. + * @see http://www.roughtype.com/archives/2010/05/experiments_in.php + * + * @return void + **/ + public function addFootnotes($articleContent) { + $footnotesWrapper = $this->dom->createElement('div'); + $footnotesWrapper->setAttribute('id', 'readability-footnotes'); + $footnotesWrapper->innerHTML = '
tags, etc.
+ *
+ * @param DOMElement
+ * @return void
+ */
+ function prepArticle($articleContent) {
+ $this->cleanStyles($articleContent);
+ $this->killBreaks($articleContent);
+ if ($this->revertForcedParagraphElements) {
+ $this->revertReadabilityStyledElements($articleContent);
+ }
+
+ /* Clean out junk from the article content */
+ $this->cleanConditionally($articleContent, 'form');
+ $this->clean($articleContent, 'object');
+ $this->clean($articleContent, 'h1');
+
+ /**
+ * If there is only one h2, they are probably using it
+ * as a header and not a subheader, so remove it since we already have a header.
+ ***/
+ if ($articleContent->getElementsByTagName('h2')->length == 1) {
+ $this->clean($articleContent, 'h2');
+ }
+ $this->clean($articleContent, 'iframe');
+
+ $this->cleanHeaders($articleContent);
+
+ /* Do these last as the previous stuff may have removed junk that will affect these */
+ $this->cleanConditionally($articleContent, 'table');
+ $this->cleanConditionally($articleContent, 'ul');
+ $this->cleanConditionally($articleContent, 'div');
+
+ /* Remove extra paragraphs */
+ $articleParagraphs = $articleContent->getElementsByTagName('p');
+ for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
+ {
+ $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
+ $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
+ $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
+
+ if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
+ {
+ $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
+ }
+ }
+
+ try {
+ $articleContent->innerHTML = preg_replace('/
]*>\s*
innerHTML);
+ //articleContent.innerHTML = articleContent.innerHTML.replace(/
]*>\s*
dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
+ }
+ }
+
+ /**
+ * Initialize a node with the readability object. Also checks the
+ * className/id for special names to add to its score.
+ *
+ * @param Element
+ * @return void
+ **/
+ protected function initializeNode($node) {
+ $readability = $this->dom->createAttribute('readability');
+ $readability->value = 0; // this is our contentScore
+ $node->setAttributeNode($readability);
+
+ switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
+ case 'DIV':
+ $readability->value += 5;
+ break;
+
+ case 'PRE':
+ case 'TD':
+ case 'BLOCKQUOTE':
+ $readability->value += 3;
+ break;
+
+ case 'ADDRESS':
+ case 'OL':
+ case 'UL':
+ case 'DL':
+ case 'DD':
+ case 'DT':
+ case 'LI':
+ case 'FORM':
+ $readability->value -= 3;
+ break;
+
+ case 'H1':
+ case 'H2':
+ case 'H3':
+ case 'H4':
+ case 'H5':
+ case 'H6':
+ case 'TH':
+ $readability->value -= 5;
+ break;
+ }
+ $readability->value += $this->getClassWeight($node);
+ }
+
+ /***
+ * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
+ * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
+ *
+ * @return DOMElement
+ **/
+ protected function grabArticle($page=null) {
+ $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
+ if (!$page) $page = $this->dom;
+ $allElements = $page->getElementsByTagName('*');
+ /**
+ * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
+ * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
+ *
+ * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
+ * TODO: Shouldn't this be a reverse traversal?
+ **/
+ $node = null;
+ $nodesToScore = array();
+ for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
+ //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
+ //$node = $targetList->item($nodeIndex);
+ $tagName = strtoupper($node->tagName);
+ /* Remove unlikely candidates */
+ if ($stripUnlikelyCandidates) {
+ $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
+ if (
+ preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
+ !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
+ $tagName != 'BODY'
+ )
+ {
+ $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
+ //$nodesToRemove[] = $node;
+ $node->parentNode->removeChild($node);
+ $nodeIndex--;
+ continue;
+ }
+ }
+
+ if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
+ $nodesToScore[] = $node;
+ }
+
+ /* Turn all divs that don't have children block level elements into p's */
+ if ($tagName == 'DIV') {
+ if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
+ //$this->dbg('Altering div to p');
+ $newNode = $this->dom->createElement('p');
+ try {
+ $newNode->innerHTML = $node->innerHTML;
+ //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
+ $node->parentNode->replaceChild($newNode, $node);
+ $nodeIndex--;
+ $nodesToScore[] = $node; // or $newNode?
+ }
+ catch(Exception $e) {
+ $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
+ }
+ }
+ else
+ {
+ /* EXPERIMENTAL */
+ // TODO: change these p elements back to text nodes after processing
+ for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
+ $childNode = $node->childNodes->item($i);
+ if ($childNode->nodeType == 3) { // XML_TEXT_NODE
+ //$this->dbg('replacing text node with a p tag with the same content.');
+ $p = $this->dom->createElement('p');
+ $p->innerHTML = $childNode->nodeValue;
+ $p->setAttribute('style', 'display: inline;');
+ $p->setAttribute('class', 'readability-styled');
+ $childNode->parentNode->replaceChild($p, $childNode);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Loop through all paragraphs, and assign a score to them based on how content-y they look.
+ * Then add their score to their parent node.
+ *
+ * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
+ **/
+ $candidates = array();
+ for ($pt=0; $pt < count($nodesToScore); $pt++) {
+ $parentNode = $nodesToScore[$pt]->parentNode;
+ // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
+ $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
+ $innerText = $this->getInnerText($nodesToScore[$pt]);
+
+ if (!$parentNode || !isset($parentNode->tagName)) {
+ continue;
+ }
+
+ /* If this paragraph is less than 25 characters, don't even count it. */
+ if(strlen($innerText) < 25) {
+ continue;
+ }
+
+ /* Initialize readability data for the parent. */
+ if (!$parentNode->hasAttribute('readability'))
+ {
+ $this->initializeNode($parentNode);
+ $candidates[] = $parentNode;
+ }
+
+ /* Initialize readability data for the grandparent. */
+ if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
+ {
+ $this->initializeNode($grandParentNode);
+ $candidates[] = $grandParentNode;
+ }
+
+ $contentScore = 0;
+
+ /* Add a point for the paragraph itself as a base. */
+ $contentScore++;
+
+ /* Add points for any commas within this paragraph */
+ $contentScore += count(explode(',', $innerText));
+
+ /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
+ $contentScore += min(floor(strlen($innerText) / 100), 3);
+
+ /* Add the score to the parent. The grandparent gets half. */
+ $parentNode->getAttributeNode('readability')->value += $contentScore;
+
+ if ($grandParentNode) {
+ $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
+ }
+ }
+
+ /**
+ * After we've calculated scores, loop through all of the possible candidate nodes we found
+ * and find the one with the highest score.
+ **/
+ $topCandidate = null;
+ for ($c=0, $cl=count($candidates); $c < $cl; $c++)
+ {
+ /**
+ * Scale the final candidates score based on link density. Good content should have a
+ * relatively small link density (5% or less) and be mostly unaffected by this operation.
+ **/
+ $readability = $candidates[$c]->getAttributeNode('readability');
+ $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
+
+ $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
+
+ if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
+ $topCandidate = $candidates[$c];
+ }
+ }
+
+ /**
+ * If we still have no top candidate, just use the body as a last resort.
+ * We also have to copy the body node so it is something we can modify.
+ **/
+ if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
+ {
+ $topCandidate = $this->dom->createElement('div');
+ $topCandidate->innerHTML = ($page instanceof DOMDocument) ? $page->saveXML($page->documentElement) : $page->innerHTML;
+ $page->innerHTML = '';
+ $page->appendChild($topCandidate);
+ $this->initializeNode($topCandidate);
+ }
+
+ /**
+ * Now that we have the top candidate, look through its siblings for content that might also be related.
+ * Things like preambles, content split by ads that we removed, etc.
+ **/
+ $articleContent = $this->dom->createElement('div');
+ $articleContent->setAttribute('id', 'readability-content');
+ $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
+ $siblingNodes = $topCandidate->parentNode->childNodes;
+
+ for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
+ {
+ $siblingNode = $siblingNodes->item($s);
+ $append = false;
+
+ $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
+
+ //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
+
+ if ($siblingNode === $topCandidate)
+ // or if ($siblingNode->isSameNode($topCandidate))
+ {
+ $append = true;
+ }
+
+ $contentBonus = 0;
+ /* Give a bonus if sibling nodes and top candidates have the example same classname */
+ if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
+ $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
+ }
+
+ if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
+ {
+ $append = true;
+ }
+
+ if (strtoupper($siblingNode->nodeName) == 'P') {
+ $linkDensity = $this->getLinkDensity($siblingNode);
+ $nodeContent = $this->getInnerText($siblingNode);
+ $nodeLength = strlen($nodeContent);
+
+ if ($nodeLength > 80 && $linkDensity < 0.25)
+ {
+ $append = true;
+ }
+ else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
+ {
+ $append = true;
+ }
+ }
+
+ if ($append)
+ {
+ $this->dbg('Appending node: ' . $siblingNode->nodeName);
+
+ $nodeToAppend = null;
+ $sibNodeName = strtoupper($siblingNode->nodeName);
+ if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
+ /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
+
+ $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
+ $nodeToAppend = $this->dom->createElement('div');
+ try {
+ $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
+ $nodeToAppend->innerHTML = $siblingNode->innerHTML;
+ }
+ catch(Exception $e)
+ {
+ $this->dbg('Could not alter siblingNode to div, reverting back to original.');
+ $nodeToAppend = $siblingNode;
+ $s--;
+ $sl--;
+ }
+ } else {
+ $nodeToAppend = $siblingNode;
+ $s--;
+ $sl--;
+ }
+
+ /* To ensure a node does not interfere with readability styles, remove its classnames */
+ $nodeToAppend->removeAttribute('class');
+
+ /* Append sibling and subtract from our list because it removes the node when you append to another node */
+ $articleContent->appendChild($nodeToAppend);
+ }
+ }
+
+ /**
+ * So we have all of the content that we need. Now we clean it up for presentation.
+ **/
+ $this->prepArticle($articleContent);
+
+ /**
+ * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
+ * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
+ * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
+ * finding the -right- content.
+ **/
+ if (strlen($this->getInnerText($articleContent, false)) < 250)
+ {
+ $this->body->innerHTML = $this->bodyCache;
+
+ if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
+ $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
+ return $this->grabArticle($this->body);
+ }
+ else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
+ $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
+ return $this->grabArticle($this->body);
+ }
+ else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
+ $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
+ return $this->grabArticle($this->body);
+ }
+ else {
+ return false;
+ }
+ }
+ return $articleContent;
+ }
+
+ /**
+ * Remove script tags from document
+ *
+ * @param DOMElement
+ * @return void
+ */
+ public function removeScripts($doc) {
+ $scripts = $doc->getElementsByTagName('script');
+ for($i = $scripts->length-1; $i >= 0; $i--)
+ {
+ $scripts->item($i)->parentNode->removeChild($scripts->item($i));
+ }
+ }
+
+ /**
+ * Get the inner text of a node.
+ * This also strips out any excess whitespace to be found.
+ *
+ * @param DOMElement $
+ * @param boolean $normalizeSpaces (default: true)
+ * @return string
+ **/
+ public function getInnerText($e, $normalizeSpaces=true) {
+ $textContent = '';
+
+ if (!isset($e->textContent) || $e->textContent == '') {
+ return '';
+ }
+
+ $textContent = trim($e->textContent);
+
+ if ($normalizeSpaces) {
+ return preg_replace($this->regexps['normalize'], ' ', $textContent);
+ } else {
+ return $textContent;
+ }
+ }
+
+ /**
+ * Get the number of times a string $s appears in the node $e.
+ *
+ * @param DOMElement $e
+ * @param string - what to count. Default is ","
+ * @return number (integer)
+ **/
+ public function getCharCount($e, $s=',') {
+ return substr_count($this->getInnerText($e), $s);
+ }
+
+ /**
+ * Remove the style attribute on every $e and under.
+ *
+ * @param DOMElement $e
+ * @return void
+ */
+ public function cleanStyles($e) {
+ $elems = $e->getElementsByTagName('*');
+ foreach ($elems as $elem) {
+ $elem->removeAttribute('style');
+ }
+ }
+
+ /**
+ * Get the density of links as a percentage of the content
+ * This is the amount of text that is inside a link divided by the total text in the node.
+ *
+ * @param DOMElement $e
+ * @return number (float)
+ */
+ public function getLinkDensity($e) {
+ $links = $e->getElementsByTagName('a');
+ $textLength = strlen($this->getInnerText($e));
+ $linkLength = 0;
+ for ($i=0, $il=$links->length; $i < $il; $i++)
+ {
+ $linkLength += strlen($this->getInnerText($links->item($i)));
+ }
+ if ($textLength > 0) {
+ return $linkLength / $textLength;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Get an elements class/id weight. Uses regular expressions to tell if this
+ * element looks good or bad.
+ *
+ * @param DOMElement $e
+ * @return number (Integer)
+ */
+ public function getClassWeight($e) {
+ if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
+ return 0;
+ }
+
+ $weight = 0;
+
+ /* Look for a special classname */
+ if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
+ {
+ if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
+ $weight -= 25;
+ }
+ if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
+ $weight += 25;
+ }
+ }
+
+ /* Look for a special ID */
+ if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
+ {
+ if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
+ $weight -= 25;
+ }
+ if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
+ $weight += 25;
+ }
+ }
+ return $weight;
+ }
+
+ /**
+ * Remove extraneous break tags from a node.
+ *
+ * @param DOMElement $node
+ * @return void
+ */
+ public function killBreaks($node) {
+ $html = $node->innerHTML;
+ $html = preg_replace($this->regexps['killBreaks'], ' Para 1 Para 2 Para 1 Para 2 The story begins...
', $html);
+ $node->innerHTML = $html;
+ }
+
+ /**
+ * Clean a node of all elements of type "tag".
+ * (Unless it's a youtube/vimeo video. People love movies.)
+ *
+ * @param DOMElement $e
+ * @param string $tag
+ * @return void
+ */
+ public function clean($e, $tag) {
+ $targetList = $e->getElementsByTagName($tag);
+ $isEmbed = ($tag == 'object' || $tag == 'embed');
+
+ for ($y=$targetList->length-1; $y >= 0; $y--) {
+ /* Allow youtube and vimeo videos through as people usually want to see those. */
+ if ($isEmbed) {
+ $attributeValues = '';
+ for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
+ $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
+ }
+
+ /* First, check the elements attributes to see if any of them contain youtube or vimeo */
+ if (preg_match($this->regexps['video'], $attributeValues)) {
+ continue;
+ }
+
+ /* Then check the elements inside this element for the same. */
+ if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
+ continue;
+ }
+ }
+ $targetList->item($y)->parentNode->removeChild($targetList->item($y));
+ }
+ }
+
+ /**
+ * Clean an element of all tags of type "tag" if they look fishy.
+ * "Fishy" is an algorithm based on content length, classnames,
+ * link density, number of images & embeds, etc.
+ *
+ * @param DOMElement $e
+ * @param string $tag
+ * @return void
+ */
+ public function cleanConditionally($e, $tag) {
+ if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
+ return;
+ }
+
+ $tagsList = $e->getElementsByTagName($tag);
+ $curTagsLength = $tagsList->length;
+
+ /**
+ * Gather counts for other typical elements embedded within.
+ * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
+ *
+ * TODO: Consider taking into account original contentScore here.
+ */
+ for ($i=$curTagsLength-1; $i >= 0; $i--) {
+ $weight = $this->getClassWeight($tagsList->item($i));
+ $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
+
+ $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
+
+ if ($weight + $contentScore < 0) {
+ $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
+ }
+ else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
+ /**
+ * If there are not very many commas, and the number of
+ * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
+ **/
+ $p = $tagsList->item($i)->getElementsByTagName('p')->length;
+ $img = $tagsList->item($i)->getElementsByTagName('img')->length;
+ $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
+ $input = $tagsList->item($i)->getElementsByTagName('input')->length;
+
+ $embedCount = 0;
+ $embeds = $tagsList->item($i)->getElementsByTagName('embed');
+ for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
+ if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
+ $embedCount++;
+ }
+ }
+
+ $linkDensity = $this->getLinkDensity($tagsList->item($i));
+ $contentLength = strlen($this->getInnerText($tagsList->item($i)));
+ $toRemove = false;
+
+ if ( $img > $p ) {
+ $toRemove = true;
+ } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
+ $toRemove = true;
+ } else if ( $input > floor($p/3) ) {
+ $toRemove = true;
+ } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
+ $toRemove = true;
+ } else if($weight < 25 && $linkDensity > 0.2) {
+ $toRemove = true;
+ } else if($weight >= 25 && $linkDensity > 0.5) {
+ $toRemove = true;
+ } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
+ $toRemove = true;
+ }
+
+ if ($toRemove) {
+ $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
+ }
+ }
+ }
+ }
+
+ /**
+ * Clean out spurious headers from an Element. Checks things like classnames and link density.
+ *
+ * @param DOMElement $e
+ * @return void
+ */
+ public function cleanHeaders($e) {
+ for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
+ $headers = $e->getElementsByTagName('h' . $headerIndex);
+ for ($i=$headers->length-1; $i >=0; $i--) {
+ if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
+ $headers->item($i)->parentNode->removeChild($headers->item($i));
+ }
+ }
+ }
+ }
+
+ public function flagIsActive($flag) {
+ return ($this->flags & $flag) > 0;
+ }
+
+ public function addFlag($flag) {
+ $this->flags = $this->flags | $flag;
+ }
+
+ public function removeFlag($flag) {
+ $this->flags = $this->flags & ~$flag;
+ }
+}
+?>
\ No newline at end of file
--
cgit v1.2.3
From 8ae45e7fe27ab416cf2a30cd2c319940037b7cbf Mon Sep 17 00:00:00 2001
From: Thomas Citharel Chapter 2
]*>[ \n\r\t]*){2,}/i',
- 'replaceFonts' => '/<(\/?)font[^>]*>/i',
- // 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
- 'normalize' => '/\s{2,}/',
- 'killBreaks' => '/(
(\s| ?)*){1,}/',
- 'video' => '/http:\/\/(www\.)?(youtube|vimeo)\.com/i',
- 'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
- );
-
- /* constants */
- const FLAG_STRIP_UNLIKELYS = 1;
- const FLAG_WEIGHT_CLASSES = 2;
- const FLAG_CLEAN_CONDITIONALLY = 4;
-
- /**
- * Create instance of Readability
- * @param string UTF-8 encoded string
- * @param string (optional) URL associated with HTML (used for footnotes)
- */
- function __construct($html, $url=null)
- {
- /* Turn all double br's into p's */
- /* Note, this is pretty costly as far as processing goes. Maybe optimize later. */
- $html = preg_replace($this->regexps['replaceBrs'], '
', $html); - $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); - $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); - $this->dom = new DOMDocument(); - $this->dom->preserveWhiteSpace = false; - $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); - @$this->dom->loadHTML($html); - $this->url = $url; - } - - /** - * Get article title element - * @return DOMElement - */ - public function getTitle() { - return $this->articleTitle; - } - - /** - * Get article content element - * @return DOMElement - */ - public function getContent() { - return $this->articleContent; - } - - /** - * Runs readability. - * - * Workflow: - * 1. Prep the document by removing script tags, css, etc. - * 2. Build readability's DOM tree. - * 3. Grab the article content from the current dom tree. - * 4. Replace the current DOM tree with the new one. - * 5. Read peacefully. - * - * @return boolean true if we found content, false otherwise - **/ - public function init() - { - $this->removeScripts($this->dom); - - // Assume successful outcome - $this->success = true; - - $bodyElems = $this->dom->getElementsByTagName('body'); - if ($bodyElems->length > 0) { - if ($this->bodyCache == null) { - $this->bodyCache = $bodyElems->item(0)->innerHTML; - } - if ($this->body == null) { - $this->body = $bodyElems->item(0); - } - } - - $this->prepDocument(); - - //die($this->dom->documentElement->parentNode->nodeType); - //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); - //die($this->getInnerHTML($this->dom->documentElement)); - - /* Build readability's DOM tree */ - $overlay = $this->dom->createElement('div'); - $innerDiv = $this->dom->createElement('div'); - $articleTitle = $this->getArticleTitle(); - $articleContent = $this->grabArticle(); - - if (!$articleContent) { - $this->success = false; - $articleContent = $this->dom->createElement('div'); - $articleContent->setAttribute('id', 'readability-content'); - $articleContent->innerHTML = '
Sorry, Readability was unable to parse this page for content.
'; - } - - $overlay->setAttribute('id', 'readOverlay'); - $innerDiv->setAttribute('id', 'readInner'); - - /* Glue the structure of our document together. */ - $innerDiv->appendChild($articleTitle); - $innerDiv->appendChild($articleContent); - $overlay->appendChild($innerDiv); - - /* Clear the old HTML, insert the new content. */ - $this->body->innerHTML = ''; - $this->body->appendChild($overlay); - //document.body.insertBefore(overlay, document.body.firstChild); - $this->body->removeAttribute('style'); - - $this->postProcessContent($articleContent); - - // Set title and content instance variables - $this->articleTitle = $articleTitle; - $this->articleContent = $articleContent; - - return $this->success; - } - - /** - * Debug - */ - protected function dbg($msg) { - if ($this->debug) echo '* ',$msg, '').replace(readability.regexps.replaceFonts, '<$1span>'); - // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. - // Manipulating innerHTML as it's done in JS is not possible in PHP. - } - - /** - * For easier reading, convert this document to have footnotes at the bottom rather than inline links. - * @see http://www.roughtype.com/archives/2010/05/experiments_in.php - * - * @return void - **/ - public function addFootnotes($articleContent) { - $footnotesWrapper = $this->dom->createElement('div'); - $footnotesWrapper->setAttribute('id', 'readability-footnotes'); - $footnotesWrapper->innerHTML = '
tags, etc.
- *
- * @param DOMElement
- * @return void
- */
- function prepArticle($articleContent) {
- $this->cleanStyles($articleContent);
- $this->killBreaks($articleContent);
- if ($this->revertForcedParagraphElements) {
- $this->revertReadabilityStyledElements($articleContent);
- }
-
- /* Clean out junk from the article content */
- $this->cleanConditionally($articleContent, 'form');
- $this->clean($articleContent, 'object');
- $this->clean($articleContent, 'h1');
-
- /**
- * If there is only one h2, they are probably using it
- * as a header and not a subheader, so remove it since we already have a header.
- ***/
- if ($articleContent->getElementsByTagName('h2')->length == 1) {
- $this->clean($articleContent, 'h2');
- }
- $this->clean($articleContent, 'iframe');
-
- $this->cleanHeaders($articleContent);
-
- /* Do these last as the previous stuff may have removed junk that will affect these */
- $this->cleanConditionally($articleContent, 'table');
- $this->cleanConditionally($articleContent, 'ul');
- $this->cleanConditionally($articleContent, 'div');
-
- /* Remove extra paragraphs */
- $articleParagraphs = $articleContent->getElementsByTagName('p');
- for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
- {
- $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
- $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
- $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
-
- if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
- {
- $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
- }
- }
-
- try {
- $articleContent->innerHTML = preg_replace('/
]*>\s*
innerHTML);
- //articleContent.innerHTML = articleContent.innerHTML.replace(/
]*>\s*
dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
- }
- }
-
- /**
- * Initialize a node with the readability object. Also checks the
- * className/id for special names to add to its score.
- *
- * @param Element
- * @return void
- **/
- protected function initializeNode($node) {
- $readability = $this->dom->createAttribute('readability');
- $readability->value = 0; // this is our contentScore
- $node->setAttributeNode($readability);
-
- switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
- case 'DIV':
- $readability->value += 5;
- break;
-
- case 'PRE':
- case 'TD':
- case 'BLOCKQUOTE':
- $readability->value += 3;
- break;
-
- case 'ADDRESS':
- case 'OL':
- case 'UL':
- case 'DL':
- case 'DD':
- case 'DT':
- case 'LI':
- case 'FORM':
- $readability->value -= 3;
- break;
-
- case 'H1':
- case 'H2':
- case 'H3':
- case 'H4':
- case 'H5':
- case 'H6':
- case 'TH':
- $readability->value -= 5;
- break;
- }
- $readability->value += $this->getClassWeight($node);
- }
-
- /***
- * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
- * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
- *
- * @return DOMElement
- **/
- protected function grabArticle($page=null) {
- $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
- if (!$page) $page = $this->dom;
- $allElements = $page->getElementsByTagName('*');
- /**
- * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
- * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
- *
- * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
- * TODO: Shouldn't this be a reverse traversal?
- **/
- $node = null;
- $nodesToScore = array();
- for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
- //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
- //$node = $targetList->item($nodeIndex);
- $tagName = strtoupper($node->tagName);
- /* Remove unlikely candidates */
- if ($stripUnlikelyCandidates) {
- $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
- if (
- preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
- !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
- $tagName != 'BODY'
- )
- {
- $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
- //$nodesToRemove[] = $node;
- $node->parentNode->removeChild($node);
- $nodeIndex--;
- continue;
- }
- }
-
- if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
- $nodesToScore[] = $node;
- }
-
- /* Turn all divs that don't have children block level elements into p's */
- if ($tagName == 'DIV') {
- if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
- //$this->dbg('Altering div to p');
- $newNode = $this->dom->createElement('p');
- try {
- $newNode->innerHTML = $node->innerHTML;
- //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
- $node->parentNode->replaceChild($newNode, $node);
- $nodeIndex--;
- $nodesToScore[] = $node; // or $newNode?
- }
- catch(Exception $e) {
- $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
- }
- }
- else
- {
- /* EXPERIMENTAL */
- // TODO: change these p elements back to text nodes after processing
- for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
- $childNode = $node->childNodes->item($i);
- if ($childNode->nodeType == 3) { // XML_TEXT_NODE
- //$this->dbg('replacing text node with a p tag with the same content.');
- $p = $this->dom->createElement('p');
- $p->innerHTML = $childNode->nodeValue;
- $p->setAttribute('style', 'display: inline;');
- $p->setAttribute('class', 'readability-styled');
- $childNode->parentNode->replaceChild($p, $childNode);
- }
- }
- }
- }
- }
-
- /**
- * Loop through all paragraphs, and assign a score to them based on how content-y they look.
- * Then add their score to their parent node.
- *
- * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
- **/
- $candidates = array();
- for ($pt=0; $pt < count($nodesToScore); $pt++) {
- $parentNode = $nodesToScore[$pt]->parentNode;
- // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
- $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
- $innerText = $this->getInnerText($nodesToScore[$pt]);
-
- if (!$parentNode || !isset($parentNode->tagName)) {
- continue;
- }
-
- /* If this paragraph is less than 25 characters, don't even count it. */
- if(strlen($innerText) < 25) {
- continue;
- }
-
- /* Initialize readability data for the parent. */
- if (!$parentNode->hasAttribute('readability'))
- {
- $this->initializeNode($parentNode);
- $candidates[] = $parentNode;
- }
-
- /* Initialize readability data for the grandparent. */
- if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
- {
- $this->initializeNode($grandParentNode);
- $candidates[] = $grandParentNode;
- }
-
- $contentScore = 0;
-
- /* Add a point for the paragraph itself as a base. */
- $contentScore++;
-
- /* Add points for any commas within this paragraph */
- $contentScore += count(explode(',', $innerText));
-
- /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
- $contentScore += min(floor(strlen($innerText) / 100), 3);
-
- /* Add the score to the parent. The grandparent gets half. */
- $parentNode->getAttributeNode('readability')->value += $contentScore;
-
- if ($grandParentNode) {
- $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
- }
- }
-
- /**
- * After we've calculated scores, loop through all of the possible candidate nodes we found
- * and find the one with the highest score.
- **/
- $topCandidate = null;
- for ($c=0, $cl=count($candidates); $c < $cl; $c++)
- {
- /**
- * Scale the final candidates score based on link density. Good content should have a
- * relatively small link density (5% or less) and be mostly unaffected by this operation.
- **/
- $readability = $candidates[$c]->getAttributeNode('readability');
- $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
-
- $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
-
- if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
- $topCandidate = $candidates[$c];
- }
- }
-
- /**
- * If we still have no top candidate, just use the body as a last resort.
- * We also have to copy the body node so it is something we can modify.
- **/
- if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
- {
- $topCandidate = $this->dom->createElement('div');
- $topCandidate->innerHTML = ($page instanceof DOMDocument) ? $page->saveXML($page->documentElement) : $page->innerHTML;
- $page->innerHTML = '';
- $page->appendChild($topCandidate);
- $this->initializeNode($topCandidate);
- }
-
- /**
- * Now that we have the top candidate, look through its siblings for content that might also be related.
- * Things like preambles, content split by ads that we removed, etc.
- **/
- $articleContent = $this->dom->createElement('div');
- $articleContent->setAttribute('id', 'readability-content');
- $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
- $siblingNodes = $topCandidate->parentNode->childNodes;
-
- for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
- {
- $siblingNode = $siblingNodes->item($s);
- $append = false;
-
- $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
-
- //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
-
- if ($siblingNode === $topCandidate)
- // or if ($siblingNode->isSameNode($topCandidate))
- {
- $append = true;
- }
-
- $contentBonus = 0;
- /* Give a bonus if sibling nodes and top candidates have the example same classname */
- if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
- $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
- }
-
- if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
- {
- $append = true;
- }
-
- if (strtoupper($siblingNode->nodeName) == 'P') {
- $linkDensity = $this->getLinkDensity($siblingNode);
- $nodeContent = $this->getInnerText($siblingNode);
- $nodeLength = strlen($nodeContent);
-
- if ($nodeLength > 80 && $linkDensity < 0.25)
- {
- $append = true;
- }
- else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
- {
- $append = true;
- }
- }
-
- if ($append)
- {
- $this->dbg('Appending node: ' . $siblingNode->nodeName);
-
- $nodeToAppend = null;
- $sibNodeName = strtoupper($siblingNode->nodeName);
- if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
- /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
-
- $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
- $nodeToAppend = $this->dom->createElement('div');
- try {
- $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
- $nodeToAppend->innerHTML = $siblingNode->innerHTML;
- }
- catch(Exception $e)
- {
- $this->dbg('Could not alter siblingNode to div, reverting back to original.');
- $nodeToAppend = $siblingNode;
- $s--;
- $sl--;
- }
- } else {
- $nodeToAppend = $siblingNode;
- $s--;
- $sl--;
- }
-
- /* To ensure a node does not interfere with readability styles, remove its classnames */
- $nodeToAppend->removeAttribute('class');
-
- /* Append sibling and subtract from our list because it removes the node when you append to another node */
- $articleContent->appendChild($nodeToAppend);
- }
- }
-
- /**
- * So we have all of the content that we need. Now we clean it up for presentation.
- **/
- $this->prepArticle($articleContent);
-
- /**
- * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
- * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
- * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
- * finding the -right- content.
- **/
- if (strlen($this->getInnerText($articleContent, false)) < 250)
- {
- $this->body->innerHTML = $this->bodyCache;
-
- if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
- $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
- return $this->grabArticle($this->body);
- }
- else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
- $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
- return $this->grabArticle($this->body);
- }
- else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
- $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
- return $this->grabArticle($this->body);
- }
- else {
- return false;
- }
- }
- return $articleContent;
- }
-
- /**
- * Remove script tags from document
- *
- * @param DOMElement
- * @return void
- */
- public function removeScripts($doc) {
- $scripts = $doc->getElementsByTagName('script');
- for($i = $scripts->length-1; $i >= 0; $i--)
- {
- $scripts->item($i)->parentNode->removeChild($scripts->item($i));
- }
- }
-
- /**
- * Get the inner text of a node.
- * This also strips out any excess whitespace to be found.
- *
- * @param DOMElement $
- * @param boolean $normalizeSpaces (default: true)
- * @return string
- **/
- public function getInnerText($e, $normalizeSpaces=true) {
- $textContent = '';
-
- if (!isset($e->textContent) || $e->textContent == '') {
- return '';
- }
-
- $textContent = trim($e->textContent);
-
- if ($normalizeSpaces) {
- return preg_replace($this->regexps['normalize'], ' ', $textContent);
- } else {
- return $textContent;
- }
- }
-
- /**
- * Get the number of times a string $s appears in the node $e.
- *
- * @param DOMElement $e
- * @param string - what to count. Default is ","
- * @return number (integer)
- **/
- public function getCharCount($e, $s=',') {
- return substr_count($this->getInnerText($e), $s);
- }
-
- /**
- * Remove the style attribute on every $e and under.
- *
- * @param DOMElement $e
- * @return void
- */
- public function cleanStyles($e) {
- $elems = $e->getElementsByTagName('*');
- foreach ($elems as $elem) {
- $elem->removeAttribute('style');
- }
- }
-
- /**
- * Get the density of links as a percentage of the content
- * This is the amount of text that is inside a link divided by the total text in the node.
- *
- * @param DOMElement $e
- * @return number (float)
- */
- public function getLinkDensity($e) {
- $links = $e->getElementsByTagName('a');
- $textLength = strlen($this->getInnerText($e));
- $linkLength = 0;
- for ($i=0, $il=$links->length; $i < $il; $i++)
- {
- $linkLength += strlen($this->getInnerText($links->item($i)));
- }
- if ($textLength > 0) {
- return $linkLength / $textLength;
- } else {
- return 0;
- }
- }
-
- /**
- * Get an elements class/id weight. Uses regular expressions to tell if this
- * element looks good or bad.
- *
- * @param DOMElement $e
- * @return number (Integer)
- */
- public function getClassWeight($e) {
- if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
- return 0;
- }
-
- $weight = 0;
-
- /* Look for a special classname */
- if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
- {
- if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
- $weight -= 25;
- }
- if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
- $weight += 25;
- }
- }
-
- /* Look for a special ID */
- if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
- {
- if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
- $weight -= 25;
- }
- if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
- $weight += 25;
- }
- }
- return $weight;
- }
-
- /**
- * Remove extraneous break tags from a node.
- *
- * @param DOMElement $node
- * @return void
- */
- public function killBreaks($node) {
- $html = $node->innerHTML;
- $html = preg_replace($this->regexps['killBreaks'], '
', $html);
- $node->innerHTML = $html;
- }
-
- /**
- * Clean a node of all elements of type "tag".
- * (Unless it's a youtube/vimeo video. People love movies.)
- *
- * @param DOMElement $e
- * @param string $tag
- * @return void
- */
- public function clean($e, $tag) {
- $targetList = $e->getElementsByTagName($tag);
- $isEmbed = ($tag == 'object' || $tag == 'embed');
-
- for ($y=$targetList->length-1; $y >= 0; $y--) {
- /* Allow youtube and vimeo videos through as people usually want to see those. */
- if ($isEmbed) {
- $attributeValues = '';
- for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
- $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
- }
-
- /* First, check the elements attributes to see if any of them contain youtube or vimeo */
- if (preg_match($this->regexps['video'], $attributeValues)) {
- continue;
- }
-
- /* Then check the elements inside this element for the same. */
- if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
- continue;
- }
- }
- $targetList->item($y)->parentNode->removeChild($targetList->item($y));
- }
- }
-
- /**
- * Clean an element of all tags of type "tag" if they look fishy.
- * "Fishy" is an algorithm based on content length, classnames,
- * link density, number of images & embeds, etc.
- *
- * @param DOMElement $e
- * @param string $tag
- * @return void
- */
- public function cleanConditionally($e, $tag) {
- if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
- return;
- }
-
- $tagsList = $e->getElementsByTagName($tag);
- $curTagsLength = $tagsList->length;
-
- /**
- * Gather counts for other typical elements embedded within.
- * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
- *
- * TODO: Consider taking into account original contentScore here.
- */
- for ($i=$curTagsLength-1; $i >= 0; $i--) {
- $weight = $this->getClassWeight($tagsList->item($i));
- $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
-
- $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
-
- if ($weight + $contentScore < 0) {
- $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
- }
- else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
- /**
- * If there are not very many commas, and the number of
- * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
- **/
- $p = $tagsList->item($i)->getElementsByTagName('p')->length;
- $img = $tagsList->item($i)->getElementsByTagName('img')->length;
- $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
- $input = $tagsList->item($i)->getElementsByTagName('input')->length;
-
- $embedCount = 0;
- $embeds = $tagsList->item($i)->getElementsByTagName('embed');
- for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
- if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
- $embedCount++;
- }
- }
-
- $linkDensity = $this->getLinkDensity($tagsList->item($i));
- $contentLength = strlen($this->getInnerText($tagsList->item($i)));
- $toRemove = false;
-
- if ( $img > $p ) {
- $toRemove = true;
- } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
- $toRemove = true;
- } else if ( $input > floor($p/3) ) {
- $toRemove = true;
- } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
- $toRemove = true;
- } else if($weight < 25 && $linkDensity > 0.2) {
- $toRemove = true;
- } else if($weight >= 25 && $linkDensity > 0.5) {
- $toRemove = true;
- } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
- $toRemove = true;
- }
-
- if ($toRemove) {
- $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
- }
- }
- }
- }
-
- /**
- * Clean out spurious headers from an Element. Checks things like classnames and link density.
- *
- * @param DOMElement $e
- * @return void
- */
- public function cleanHeaders($e) {
- for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
- $headers = $e->getElementsByTagName('h' . $headerIndex);
- for ($i=$headers->length-1; $i >=0; $i--) {
- if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
- $headers->item($i)->parentNode->removeChild($headers->item($i));
- }
- }
- }
- }
-
- public function flagIsActive($flag) {
- return ($this->flags & $flag) > 0;
- }
-
- public function addFlag($flag) {
- $this->flags = $this->flags | $flag;
- }
-
- public function removeFlag($flag) {
- $this->flags = $this->flags & ~$flag;
- }
-}
-?>
\ No newline at end of file
--
cgit v1.2.3