aboutsummaryrefslogtreecommitdiffhomepage
path: root/inc/3rdparty/libraries/language-detect/LanguageDetect.php
diff options
context:
space:
mode:
Diffstat (limited to 'inc/3rdparty/libraries/language-detect/LanguageDetect.php')
-rw-r--r--inc/3rdparty/libraries/language-detect/LanguageDetect.php1635
1 files changed, 1635 insertions, 0 deletions
diff --git a/inc/3rdparty/libraries/language-detect/LanguageDetect.php b/inc/3rdparty/libraries/language-detect/LanguageDetect.php
new file mode 100644
index 00000000..09b11546
--- /dev/null
+++ b/inc/3rdparty/libraries/language-detect/LanguageDetect.php
@@ -0,0 +1,1635 @@
1<?php
2
3/**
4 * Detects the language of a given piece of text.
5 *
6 * Attempts to detect the language of a sample of text by correlating ranked
7 * 3-gram frequencies to a table of 3-gram frequencies of known languages.
8 *
9 * Implements a version of a technique originally proposed by Cavnar & Trenkle
10 * (1994): "N-Gram-Based Text Categorization"
11 *
12 * PHP versions 4 and 5
13 *
14 * @category Text
15 * @package Text_LanguageDetect
16 * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
17 * @copyright 2005-2006 Nicholas Pisarro
18 * @license http://www.debian.org/misc/bsd.license BSD
19 * @version CVS: $Id: LanguageDetect.php,v 1.20 2008/07/01 02:09:15 taak Exp $
20 * @link http://pear.php.net/package/Text_LanguageDetect/
21 * @link http://langdetect.blogspot.com/
22 */
23
24//require_once 'PEAR.php';
25require_once 'Parser.php';
26
27/**
28 * Language detection class
29 *
30 * Requires the langauge model database (lang.dat) that should have
31 * accompanied this class definition in order to be instantiated.
32 *
33 * Example usage:
34 *
35 * <code>
36 * require_once 'Text/LanguageDetect.php';
37 *
38 * $l = new Text_LanguageDetect;
39 *
40 * $stdin = fopen('php://stdin', 'r');
41 *
42 * echo "Supported languages:\n";
43 *
44 * $langs = $l->getLanguages();
45 * if (PEAR::isError($langs)) {
46 * die($langs->getMessage());
47 * }
48 *
49 * sort($langs);
50 * echo join(', ', $langs);
51 *
52 * while ($line = fgets($stdin)) {
53 * print_r($l->detect($line, 4));
54 * }
55 * </code>
56 *
57 * @category Text
58 * @package Text_LanguageDetect
59 * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
60 * @copyright 2005 Nicholas Pisarro
61 * @license http://www.debian.org/misc/bsd.license BSD
62 * @version Release: @package_version@
63 * @todo allow users to generate their own language models
64 */
65
66class Text_LanguageDetect
67{
68 /**
69 * The filename that stores the trigram data for the detector
70 *
71 * If this value starts with a slash (/) or a dot (.) the value of
72 * $this->_data_dir will be ignored
73 *
74 * @var string
75 * @access private
76 */
77 var $_db_filename = './lang.dat';
78
79 /**
80 * The filename that stores the unicode block definitions
81 *
82 * If this value starts with a slash (/) or a dot (.) the value of
83 * $this->_data_dir will be ignored
84 *
85 * @var string
86 * @access private
87 */
88 var $_unicode_db_filename = './unicode_blocks.dat';
89
90 /**
91 * The data directory
92 *
93 * Should be set by PEAR installer
94 *
95 * @var string
96 * @access private
97 */
98 var $_data_dir = '@data_dir@';
99
100 /**
101 * The trigram data for comparison
102 *
103 * Will be loaded on start from $this->_db_filename
104 *
105 * May be set to a PEAR_Error object if there is an error during its
106 * initialization
107 *
108 * @var array
109 * @access private
110 */
111 var $_lang_db = array();
112
113 /**
114 * stores the map of the trigram data to unicode characters
115 *
116 * @access private
117 * @var array
118 */
119 var $_unicode_map;
120
121 /**
122 * The size of the trigram data arrays
123 *
124 * @var int
125 * @access private
126 */
127 var $_threshold = 300;
128
129 /**
130 * the maximum possible score.
131 *
132 * needed for score normalization. Different depending on the
133 * perl compatibility setting
134 *
135 * @access private
136 * @var int
137 * @see setPerlCompatible()
138 */
139 var $_max_score = 0;
140
141 /**
142 * Whether or not to simulate perl's Language::Guess exactly
143 *
144 * @access private
145 * @var bool
146 * @see setPerlCompatible()
147 */
148 var $_perl_compatible = false;
149
150 /**
151 * Whether to use the unicode block detection to speed up processing
152 *
153 * @access private
154 * @var bool
155 */
156 var $_use_unicode_narrowing = true;
157
158 /**
159 * stores the result of the clustering operation
160 *
161 * @access private
162 * @var array
163 * @see clusterLanguages()
164 */
165 var $_clusters;
166
167 /**
168 * Constructor
169 *
170 * Will attempt to load the language database. If it fails, you will get
171 * a PEAR_Error object returned when you try to use detect()
172 *
173 */
174 function Text_LanguageDetect($db=null, $unicode_db=null)
175 {
176 if (isset($db)) $this->_db_filename = $db;
177 if (isset($unicode_db)) $this->_unicode_db_filename = $unicode_db;
178
179 $data = $this->_readdb($this->_db_filename);
180 $this->_lang_db = $data['trigram'];
181
182 if (isset($data['trigram-unicodemap'])) {
183 $this->_unicode_map = $data['trigram-unicodemap'];
184 }
185
186 // Not yet implemented:
187 if (isset($data['trigram-clusters'])) {
188 $this->_clusters = $data['trigram-clusters'];
189 }
190 }
191
192 /**
193 * Returns the path to the location of the database
194 *
195 * @access private
196 * @return string expected path to the language model database
197 */
198 function _get_data_loc($fname)
199 {
200 return $fname;
201 }
202
203 /**
204 * Loads the language trigram database from filename
205 *
206 * Trigram datbase should be a serialize()'d array
207 *
208 * @access private
209 * @param string $fname the filename where the data is stored
210 * @return array the language model data
211 * @throws PEAR_Error
212 */
213 function _readdb($fname)
214 {
215 // finds the correct data dir
216 $fname = $this->_get_data_loc($fname);
217
218 // input check
219 if (!file_exists($fname)) {
220 throw new Exception('Language database does not exist.');
221 } elseif (!is_readable($fname)) {
222 throw new Exception('Language database is not readable.');
223 }
224
225 if (function_exists('file_get_contents')) {
226 return unserialize(file_get_contents($fname));
227 } else {
228 // if you don't have file_get_contents(),
229 // then this is the next fastest way
230 ob_start();
231 readfile($fname);
232 $contents = ob_get_contents();
233 ob_end_clean();
234 return unserialize($contents);
235 }
236 }
237
238
239 /**
240 * Checks if this object is ready to detect languages
241 *
242 * @access private
243 * @param mixed &$err error object to be returned by reference, if any
244 * @return bool true if no errors
245 */
246 function _setup_ok(&$err)
247 {
248 if (!is_array($this->_lang_db)) {
249 if (ini_get('magic_quotes_runtime')) {
250 throw new Exception('Error loading database. Try turning magic_quotes_runtime off.');
251 } else {
252 throw new Exception('Language database is not an array.');
253 }
254 return false;
255
256 } elseif (empty($this->_lang_db)) {
257 throw new Exception('Language database has no elements.');
258 return false;
259
260 } else {
261 return true;
262 }
263 }
264
265 /**
266 * Omits languages
267 *
268 * Pass this function the name of or an array of names of
269 * languages that you don't want considered
270 *
271 * If you're only expecting a limited set of languages, this can greatly
272 * speed up processing
273 *
274 * @access public
275 * @param mixed $omit_list language name or array of names to omit
276 * @param bool $include_only if true will include (rather than
277 * exclude) only those in the list
278 * @return int number of languages successfully deleted
279 * @throws PEAR_Error
280 */
281 function omitLanguages($omit_list, $include_only = false)
282 {
283
284 // setup check
285 if (!$this->_setup_ok($err)) {
286 return $err;
287 }
288
289 $deleted = 0;
290
291 // deleting the given languages
292 if (!$include_only) {
293 if (!is_array($omit_list)) {
294 $omit_list = strtolower($omit_list); // case desensitize
295 if (isset($this->_lang_db[$omit_list])) {
296 unset($this->_lang_db[$omit_list]);
297 $deleted++;
298 }
299 } else {
300 foreach ($omit_list as $omit_lang) {
301 if (isset($this->_lang_db[$omit_lang])) {
302 unset($this->_lang_db[$omit_lang]);
303 $deleted++;
304 }
305 }
306 }
307
308 // deleting all except the given languages
309 } else {
310 if (!is_array($omit_list)) {
311 $omit_list = array($omit_list);
312 }
313
314 // case desensitize
315 foreach ($omit_list as $key => $omit_lang) {
316 $omit_list[$key] = strtolower($omit_lang);
317 }
318
319 foreach (array_keys($this->_lang_db) as $lang) {
320 if (!in_array($lang, $omit_list)) {
321 unset($this->_lang_db[$lang]);
322 $deleted++;
323 }
324 }
325 }
326
327 // reset the cluster cache if the number of languages changes
328 // this will then have to be recalculated
329 if (isset($this->_clusters) && $deleted > 0) {
330 unset($this->_clusters);
331 }
332
333 return $deleted;
334 }
335
336
337 /**
338 * Returns the number of languages that this object can detect
339 *
340 * @access public
341 * @return int the number of languages
342 * @throws PEAR_Error
343 */
344 function getLanguageCount()
345 {
346 if (!$this->_setup_ok($err)) {
347 return $err;
348 } else {
349 return count($this->_lang_db);
350 }
351 }
352
353 /**
354 * Returns true if a given language exists
355 *
356 * If passed an array of names, will return true only if all exist
357 *
358 * @access public
359 * @param mixed $lang language name or array of language names
360 * @return bool true if language model exists
361 * @throws PEAR_Error
362 */
363 function languageExists($lang)
364 {
365 if (!$this->_setup_ok($err)) {
366 return $err;
367 } else {
368 // string
369 if (is_string($lang)) {
370 return isset($this->_lang_db[strtolower($lang)]);
371
372 // array
373 } elseif (is_array($lang)) {
374 foreach ($lang as $test_lang) {
375 if (!isset($this->_lang_db[strtolower($test_lang)])) {
376 return false;
377 }
378 }
379 return true;
380
381 // other (error)
382 } else {
383 throw new Exception('Unknown type passed to languageExists()');
384 }
385 }
386 }
387
388 /**
389 * Returns the list of detectable languages
390 *
391 * @access public
392 * @return array the names of the languages known to this object
393 * @throws PEAR_Error
394 */
395 function getLanguages()
396 {
397 if (!$this->_setup_ok($err)) {
398 return $err;
399 } else {
400 return array_keys($this->_lang_db);
401 }
402 }
403
404 /**
405 * Make this object behave like Language::Guess
406 *
407 * @access public
408 * @param bool $setting false to turn off perl compatibility
409 */
410 function setPerlCompatible($setting = true)
411 {
412 if (is_bool($setting)) { // input check
413 $this->_perl_compatible = $setting;
414
415 if ($setting == true) {
416 $this->_max_score = $this->_threshold;
417 } else {
418 $this->_max_score = 0;
419 }
420 }
421
422 }
423
424 /**
425 * Whether to use unicode block ranges in detection
426 *
427 * Should speed up most detections if turned on (detault is on). In some
428 * circumstances it may be slower, such as for large text samples (> 10K)
429 * in languages that use latin scripts. In other cases it should speed up
430 * detection noticeably.
431 *
432 * @access public
433 * @param bool $setting false to turn off
434 */
435 function useUnicodeBlocks($setting = true)
436 {
437 if (is_bool($setting)) {
438 $this->_use_unicode_narrowing = $setting;
439 }
440 }
441
442 /**
443 * Converts a piece of text into trigrams
444 *
445 * Superceded by the Text_LanguageDetect_Parser class
446 *
447 * @access private
448 * @param string $text text to convert
449 * @return array array of trigram frequencies
450 */
451 function _trigram($text)
452 {
453 $s = new Text_LanguageDetect_Parser($text, $this->_db_filename, $this->_unicode_db_filename);
454 $s->prepareTrigram();
455 $s->prepareUnicode(false);
456 $s->setPadStart(!$this->_perl_compatible);
457 $s->analyze();
458 return $s->getTrigramFreqs();
459 }
460
461 /**
462 * Converts a set of trigrams from frequencies to ranks
463 *
464 * Thresholds (cuts off) the list at $this->_threshold
465 *
466 * @access protected
467 * @param array $arr array of trgram
468 * @return array ranks of trigrams
469 */
470 function _arr_rank(&$arr)
471 {
472
473 // sorts alphabetically first as a standard way of breaking rank ties
474 $this->_bub_sort($arr);
475
476 // below might also work, but seemed to introduce errors in testing
477 //ksort($arr);
478 //asort($arr);
479
480 $rank = array();
481
482 $i = 0;
483 foreach ($arr as $key => $value) {
484 $rank[$key] = $i++;
485
486 // cut off at a standard threshold
487 if ($i >= $this->_threshold) {
488 break;
489 }
490 }
491
492 return $rank;
493 }
494
495 /**
496 * Sorts an array by value breaking ties alphabetically
497 *
498 * @access private
499 * @param array &$arr the array to sort
500 */
501 function _bub_sort(&$arr)
502 {
503 // should do the same as this perl statement:
504 // sort { $trigrams{$b} == $trigrams{$a} ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
505
506 // needs to sort by both key and value at once
507 // using the key to break ties for the value
508
509 // converts array into an array of arrays of each key and value
510 // may be a better way of doing this
511 $combined = array();
512
513 foreach ($arr as $key => $value) {
514 $combined[] = array($key, $value);
515 }
516
517 usort($combined, array($this, '_sort_func'));
518
519 $replacement = array();
520 foreach ($combined as $key => $value) {
521 list($new_key, $new_value) = $value;
522 $replacement[$new_key] = $new_value;
523 }
524
525 $arr = $replacement;
526 }
527
528 /**
529 * Sort function used by bubble sort
530 *
531 * Callback function for usort().
532 *
533 * @access private
534 * @param array first param passed by usort()
535 * @param array second param passed by usort()
536 * @return int 1 if $a is greater, -1 if not
537 * @see _bub_sort()
538 */
539 function _sort_func($a, $b)
540 {
541 // each is actually a key/value pair, so that it can compare using both
542 list($a_key, $a_value) = $a;
543 list($b_key, $b_value) = $b;
544
545 // if the values are the same, break ties using the key
546 if ($a_value == $b_value) {
547 return strcmp($a_key, $b_key);
548
549 // if not, just sort normally
550 } else {
551 if ($a_value > $b_value) {
552 return -1;
553 } else {
554 return 1;
555 }
556 }
557
558 // 0 should not be possible because keys must be unique
559 }
560
561 /**
562 * Calculates a linear rank-order distance statistic between two sets of
563 * ranked trigrams
564 *
565 * Sums the differences in rank for each trigram. If the trigram does not
566 * appear in both, consider it a difference of $this->_threshold.
567 *
568 * This distance measure was proposed by Cavnar & Trenkle (1994). Despite
569 * its simplicity it has been shown to be highly accurate for language
570 * identification tasks.
571 *
572 * @access private
573 * @param array $arr1 the reference set of trigram ranks
574 * @param array $arr2 the target set of trigram ranks
575 * @return int the sum of the differences between the ranks of
576 * the two trigram sets
577 */
578 function _distance(&$arr1, &$arr2)
579 {
580 $sumdist = 0;
581
582 foreach ($arr2 as $key => $value) {
583 if (isset($arr1[$key])) {
584 $distance = abs($value - $arr1[$key]);
585 } else {
586 // $this->_threshold sets the maximum possible distance value
587 // for any one pair of trigrams
588 $distance = $this->_threshold;
589 }
590 $sumdist += $distance;
591 }
592
593 return $sumdist;
594
595 // todo: there are other distance statistics to try, e.g. relative
596 // entropy, but they're probably more costly to compute
597 }
598
599 /**
600 * Normalizes the score returned by _distance()
601 *
602 * Different if perl compatible or not
603 *
604 * @access private
605 * @param int $score the score from _distance()
606 * @param int $base_count the number of trigrams being considered
607 * @return float the normalized score
608 * @see _distance()
609 */
610 function _normalize_score($score, $base_count = null)
611 {
612 if ($base_count === null) {
613 $base_count = $this->_threshold;
614 }
615
616 if (!$this->_perl_compatible) {
617 return 1 - ($score / $base_count / $this->_threshold);
618 } else {
619 return floor($score / $base_count);
620 }
621 }
622
623
624 /**
625 * Detects the closeness of a sample of text to the known languages
626 *
627 * Calculates the statistical difference between the text and
628 * the trigrams for each language, normalizes the score then
629 * returns results for all languages in sorted order
630 *
631 * If perl compatible, the score is 300-0, 0 being most similar.
632 * Otherwise, it's 0-1 with 1 being most similar.
633 *
634 * The $sample text should be at least a few sentences in length;
635 * should be ascii-7 or utf8 encoded, if another and the mbstring extension
636 * is present it will try to detect and convert. However, experience has
637 * shown that mb_detect_encoding() *does not work very well* with at least
638 * some types of encoding.
639 *
640 * @access public
641 * @param string $sample a sample of text to compare.
642 * @param int $limit if specified, return an array of the most likely
643 * $limit languages and their scores.
644 * @return mixed sorted array of language scores, blank array if no
645 * useable text was found, or PEAR_Error if error
646 * with the object setup
647 * @see _distance()
648 * @throws PEAR_Error
649 */
650 function detect($sample, $limit = 0)
651 {
652 if (!$this->_setup_ok($err)) {
653 return $err;
654 }
655
656 // input check
657 if (!Text_LanguageDetect_Parser::validateString($sample)) {
658 return array();
659 }
660
661 // check char encoding
662 // (only if mbstring extension is compiled and PHP > 4.0.6)
663 if (function_exists('mb_detect_encoding')
664 && function_exists('mb_convert_encoding')) {
665
666 // mb_detect_encoding isn't very reliable, to say the least
667 // detection should still work with a sufficient sample of ascii characters
668 $encoding = mb_detect_encoding($sample);
669
670 // mb_detect_encoding() will return FALSE if detection fails
671 // don't attempt conversion if that's the case
672 if ($encoding != 'ASCII' && $encoding != 'UTF-8' && $encoding !== false) {
673
674 if (function_exists('mb_list_encodings')) {
675
676 // verify the encoding exists in mb_list_encodings
677 if (in_array($encoding, mb_list_encodings())) {
678 $sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
679 }
680
681 // if the previous condition failed:
682 // somehow we detected an encoding that also we don't support
683
684 } else {
685 // php 4 doesnt have mb_list_encodings()
686 // so attempt with error suppression
687 $sample = @mb_convert_encoding($sample, 'UTF-8', $encoding);
688 }
689 }
690 }
691
692 $sample_obj = new Text_LanguageDetect_Parser($sample, $this->_db_filename, $this->_unicode_db_filename);
693 $sample_obj->prepareTrigram();
694 if ($this->_use_unicode_narrowing) {
695 $sample_obj->prepareUnicode();
696 }
697 $sample_obj->setPadStart(!$this->_perl_compatible);
698 $sample_obj->analyze();
699
700 $trigram_freqs =& $sample_obj->getTrigramRanks();
701 $trigram_count = count($trigram_freqs);
702
703 if ($trigram_count == 0) {
704 return array();
705 }
706
707 $scores = array();
708
709 // use unicode block detection to narrow down the possibilities
710 if ($this->_use_unicode_narrowing) {
711 $blocks =& $sample_obj->getUnicodeBlocks();
712
713 if (is_array($blocks)) {
714 $present_blocks = array_keys($blocks);
715 } else {
716 throw new Exception('Error during block detection');
717 }
718
719 $possible_langs = array();
720
721 foreach ($present_blocks as $blockname) {
722 if (isset($this->_unicode_map[$blockname])) {
723
724 $possible_langs = array_merge(
725 $possible_langs,
726 array_keys($this->_unicode_map[$blockname])
727 );
728
729 // todo: faster way to do this?
730 }
731 }
732
733 // could also try an intersect operation rather than a union
734 // in other words, choose languages whose trigrams contain
735 // ALL of the unicode blocks found in this sample
736 // would improve speed but would be completely thrown off by an
737 // unexpected character, like an umlaut appearing in english text
738
739 $possible_langs = array_intersect(
740 array_keys($this->_lang_db),
741 array_unique($possible_langs)
742 );
743
744 // needs to intersect it with the keys of _lang_db in case
745 // languages have been omitted
746
747 // or just try 'em all
748 } else {
749 $possible_langs = array_keys($this->_lang_db);
750 }
751
752
753 foreach ($possible_langs as $lang) {
754 $scores[$lang] =
755 $this->_normalize_score(
756 $this->_distance($this->_lang_db[$lang], $trigram_freqs),
757 $trigram_count);
758 }
759
760 unset($sample_obj);
761
762 if ($this->_perl_compatible) {
763 asort($scores);
764 } else {
765 arsort($scores);
766 }
767
768 // todo: drop languages with a score of $this->_max_score?
769
770 // limit the number of returned scores
771 if ($limit && is_numeric($limit)) {
772 $limited_scores = array();
773
774 $i = 0;
775
776 foreach ($scores as $key => $value) {
777 if ($i++ >= $limit) {
778 break;
779 }
780
781 $limited_scores[$key] = $value;
782 }
783
784 return $limited_scores;
785 } else {
786 return $scores;
787 }
788 }
789
790 /**
791 * Returns only the most similar language to the text sample
792 *
793 * Calls $this->detect() and returns only the top result
794 *
795 * @access public
796 * @param string $sample text to detect the language of
797 * @return string the name of the most likely language
798 * or null if no language is similar
799 * @see detect()
800 * @throws PEAR_Error
801 */
802 function detectSimple($sample)
803 {
804 $scores = $this->detect($sample, 1);
805
806 // if top language has the maximum possible score,
807 // then the top score will have been picked at random
808 if ( !is_array($scores)
809 || empty($scores)
810 || current($scores) == $this->_max_score) {
811
812 return null;
813
814 } else {
815 return ucfirst(key($scores));
816 }
817 }
818
819 /**
820 * Returns an array containing the most similar language and a confidence
821 * rating
822 *
823 * Confidence is a simple measure calculated from the similarity score
824 * minus the similarity score from the next most similar language
825 * divided by the highest possible score. Languages that have closely
826 * related cousins (e.g. Norwegian and Danish) should generally have lower
827 * confidence scores.
828 *
829 * The similarity score answers the question "How likely is the text the
830 * returned language regardless of the other languages considered?" The
831 * confidence score is one way of answering the question "how likely is the
832 * text the detected language relative to the rest of the language model
833 * set?"
834 *
835 * To see how similar languages are a priori, see languageSimilarity()
836 *
837 * @access public
838 * @param string $sample text for which language will be detected
839 * @return array most similar language, score and confidence rating
840 * or null if no language is similar
841 * @see detect()
842 * @throws PEAR_Error
843 */
844 function detectConfidence($sample)
845 {
846 $scores = $this->detect($sample, 2);
847
848 // if most similar language has the max score, it
849 // will have been picked at random
850 if ( !is_array($scores)
851 || empty($scores)
852 || current($scores) == $this->_max_score) {
853
854 return null;
855 }
856
857 $arr['language'] = ucfirst(key($scores));
858 $arr['similarity'] = current($scores);
859 if (next($scores) !== false) { // if false then no next element
860 // the goal is to return a higher value if the distance between
861 // the similarity of the first score and the second score is high
862
863 if ($this->_perl_compatible) {
864
865 $arr['confidence'] =
866 (current($scores) - $arr['similarity']) / $this->_max_score;
867
868 } else {
869
870 $arr['confidence'] = $arr['similarity'] - current($scores);
871
872 }
873
874 } else {
875 $arr['confidence'] = null;
876 }
877
878 return $arr;
879 }
880
881 /**
882 * Returns the distribution of unicode blocks in a given utf8 string
883 *
884 * For the block name of a single char, use unicodeBlockName()
885 *
886 * @access public
887 * @param string $str input string. Must be ascii or utf8
888 * @param bool $skip_symbols if true, skip ascii digits, symbols and
889 * non-printing characters. Includes spaces,
890 * newlines and common punctutation characters.
891 * @return array
892 * @throws PEAR_Error
893 */
894 function detectUnicodeBlocks($str, $skip_symbols)
895 {
896 // input check
897 if (!is_bool($skip_symbols)) {
898 throw new Exception('Second parameter must be boolean');
899 }
900
901 if (!is_string($str)) {
902 throw new Exception('First parameter was not a string');
903 }
904
905 $sample_obj = new Text_LanguageDetect_Parser($str, $this->_db_filename, $this->_unicode_db_filename);
906 $sample_obj->prepareUnicode();
907 $sample_obj->prepareTrigram(false);
908 $sample_obj->setUnicodeSkipSymbols($skip_symbols);
909 $sample_obj->analyze();
910 $blocks =& $sample_obj->getUnicodeBlocks();
911 unset($sample_obj);
912 return $blocks;
913 }
914
915 /**
916 * Returns the block name for a given unicode value
917 *
918 * If passed a string, will assume it is being passed a UTF8-formatted
919 * character and will automatically convert. Otherwise it will assume it
920 * is being passed a numeric unicode value.
921 *
922 * Make sure input is of the correct type!
923 *
924 * @access public
925 * @param mixed $unicode unicode value or utf8 char
926 * @return mixed the block name string or false if not found
927 * @throws PEAR_Error
928 */
929 function unicodeBlockName($unicode) {
930 if (is_string($unicode)) {
931 // assume it is being passed a utf8 char, so convert it
932
933 // input check
934 if ($this->utf8strlen($unicode) > 1) {
935 throw new Exception('Pass this function only a single char');
936 }
937
938 $unicode = $this->_utf8char2unicode($unicode);
939
940 if ($unicode == -1) {
941 throw new Exception('Malformatted char');
942 }
943
944 // input check
945 } elseif (!is_int($unicode)) {
946 throw new Exception('Input must be of type string or int.');
947 }
948
949 $blocks =& $this->_read_unicode_block_db();
950
951 $result = $this->_unicode_block_name($unicode, $blocks);
952
953 if ($result == -1) {
954 return false;
955 } else {
956 return $result[2];
957 }
958 }
959
960 /**
961 * Searches the unicode block database
962 *
963 * Returns the block name for a given unicode value. unicodeBlockName() is
964 * the public interface for this function, which does input checks which
965 * this function omits for speed.
966 *
967 * @access protected
968 * @param int $unicode the unicode value
969 * @param array &$blocks the block database
970 * @param int $block_count the number of defined blocks in the database
971 * @see unicodeBlockName()
972 */
973 function _unicode_block_name($unicode, &$blocks, $block_count = -1) {
974 // for a reference, see
975 // http://www.unicode.org/Public/UNIDATA/Blocks.txt
976
977 // assume that ascii characters are the most common
978 // so try it first for efficiency
979 if ($unicode <= $blocks[0][1]) {
980 return $blocks[0];
981 }
982
983 // the optional $block_count param is for efficiency
984 // so we this function doesn't have to run count() every time
985 if ($block_count != -1) {
986 $high = $block_count - 1;
987 } else {
988 $high = count($blocks) - 1;
989 }
990
991 $low = 1; // start with 1 because ascii was 0
992
993 // your average binary search algorithm
994 while ($low <= $high) {
995 $mid = floor(($low + $high) / 2);
996
997 // if it's lower than the lower bound
998 if ($unicode < $blocks[$mid][0]) {
999 $high = $mid - 1;
1000
1001 // if it's higher than the upper bound
1002 } elseif ($unicode > $blocks[$mid][1]) {
1003 $low = $mid + 1;
1004
1005 // found it
1006 } else {
1007 return $blocks[$mid];
1008 }
1009 }
1010
1011 // failed to find the block
1012 return -1;
1013
1014 // todo: differentiate when it's out of range or when it falls
1015 // into an unassigned range?
1016 }
1017
1018 /**
1019 * Brings up the unicode block database
1020 *
1021 * @access protected
1022 * @return array the database of unicode block definitions
1023 * @throws PEAR_Error
1024 */
1025 function &_read_unicode_block_db() {
1026 // since the unicode definitions are always going to be the same,
1027 // might as well share the memory for the db with all other instances
1028 // of this class
1029 static $data;
1030
1031 if (!isset($data)) {
1032 $data = $this->_readdb($this->_unicode_db_filename);
1033 }
1034
1035 return $data;
1036 }
1037
1038 /**
1039 * Calculate the similarities between the language models
1040 *
1041 * Use this function to see how similar languages are to each other.
1042 *
1043 * If passed 2 language names, will return just those languages compared.
1044 * If passed 1 language name, will return that language compared to
1045 * all others.
1046 * If passed none, will return an array of every language model compared
1047 * to every other one.
1048 *
1049 * @access public
1050 * @param string $lang1 the name of the first language to be compared
1051 * @param string $lang2 the name of the second language to be compared
1052 * @return array scores of every language compared
1053 * or the score of just the provided languages
1054 * or null if one of the supplied languages does not exist
1055 * @throws PEAR_Error
1056 */
1057 function languageSimilarity($lang1 = null, $lang2 = null)
1058 {
1059 if (!$this->_setup_ok($err)) {
1060 return $err;
1061 }
1062
1063 if ($lang1 != null) {
1064 $lang1 = strtolower($lang1);
1065
1066 // check if language model exists
1067 if (!isset($this->_lang_db[$lang1])) {
1068 return null;
1069 }
1070
1071 if ($lang2 != null) {
1072
1073 // can't only set the second param
1074 if ($lang1 == null) {
1075 return null;
1076 // check if language model exists
1077 } elseif (!isset($this->_lang_db[$lang2])) {
1078 return null;
1079 }
1080
1081 $lang2 = strtolower($lang2);
1082
1083 // compare just these two languages
1084 return $this->_normalize_score(
1085 $this->_distance(
1086 $this->_lang_db[$lang1],
1087 $this->_lang_db[$lang2]
1088 )
1089 );
1090
1091
1092 // compare just $lang1 to all languages
1093 } else {
1094 $return_arr = array();
1095 foreach ($this->_lang_db as $key => $value) {
1096 if ($key != $lang1) { // don't compare a language to itself
1097 $return_arr[$key] = $this->_normalize_score(
1098 $this->_distance($this->_lang_db[$lang1], $value));
1099 }
1100 }
1101 asort($return_arr);
1102
1103 return $return_arr;
1104 }
1105
1106
1107 // compare all languages to each other
1108 } else {
1109 $return_arr = array();
1110 foreach (array_keys($this->_lang_db) as $lang1) {
1111 foreach (array_keys($this->_lang_db) as $lang2) {
1112
1113 // skip comparing languages to themselves
1114 if ($lang1 != $lang2) {
1115
1116 // don't re-calculate what's already been done
1117 if (isset($return_arr[$lang2][$lang1])) {
1118
1119 $return_arr[$lang1][$lang2] =
1120 $return_arr[$lang2][$lang1];
1121
1122 // calculate
1123 } else {
1124
1125 $return_arr[$lang1][$lang2] =
1126 $this->_normalize_score(
1127 $this->_distance(
1128 $this->_lang_db[$lang1],
1129 $this->_lang_db[$lang2]
1130 )
1131 );
1132
1133 }
1134 }
1135 }
1136 }
1137 return $return_arr;
1138 }
1139 }
1140
1141 /**
1142 * Cluster known languages according to languageSimilarity()
1143 *
1144 * WARNING: this method is EXPERIMENTAL. It is not recommended for common
1145 * use, and it may disappear or its functionality may change in future
1146 * releases without notice.
1147 *
1148 * Uses a nearest neighbor technique to generate the maximum possible
1149 * number of dendograms from the similarity data.
1150 *
1151 * @access public
1152 * @return array language cluster data
1153 * @throws PEAR_Error
1154 * @see languageSimilarity()
1155 * @deprecated this function will eventually be removed and placed into
1156 * the model generation class
1157 */
1158 function clusterLanguages()
1159 {
1160 // todo: set the maximum number of clusters
1161
1162 // setup check
1163 if (!$this->_setup_ok($err)) {
1164 return $err;
1165 }
1166
1167 // return cached result, if any
1168 if (isset($this->_clusters)) {
1169 return $this->_clusters;
1170 }
1171
1172 $langs = array_keys($this->_lang_db);
1173
1174 $arr = $this->languageSimilarity();
1175
1176 sort($langs);
1177
1178 foreach ($langs as $lang) {
1179 if (!isset($this->_lang_db[$lang])) {
1180 throw new Exception("missing $lang!\n");
1181 }
1182 }
1183
1184 // http://www.psychstat.missouristate.edu/multibook/mlt04m.html
1185 foreach ($langs as $old_key => $lang1) {
1186 $langs[$lang1] = $lang1;
1187 unset($langs[$old_key]);
1188 }
1189
1190 $i = 0;
1191 while (count($langs) > 2 && $i++ < 200) {
1192 $highest_score = -1;
1193 $highest_key1 = '';
1194 $highest_key2 = '';
1195 foreach ($langs as $lang1) {
1196 foreach ($langs as $lang2) {
1197 if ( $lang1 != $lang2
1198 && $arr[$lang1][$lang2] > $highest_score) {
1199 $highest_score = $arr[$lang1][$lang2];
1200 $highest_key1 = $lang1;
1201 $highest_key2 = $lang2;
1202 }
1203 }
1204 }
1205
1206 if (!$highest_key1) {
1207 // should not ever happen
1208 throw new Exception("no highest key? (step: $i)");
1209 }
1210
1211 if ($highest_score == 0) {
1212 // languages are perfectly dissimilar
1213 break;
1214 }
1215
1216 // $highest_key1 and $highest_key2 are most similar
1217 $sum1 = array_sum($arr[$highest_key1]);
1218 $sum2 = array_sum($arr[$highest_key2]);
1219
1220 // use the score for the one that is most similar to the rest of
1221 // the field as the score for the group
1222 // todo: could try averaging or "centroid" method instead
1223 // seems like that might make more sense
1224 // actually nearest neighbor may be better for binary searching
1225
1226
1227 // for "Complete Linkage"/"furthest neighbor"
1228 // sign should be <
1229 // for "Single Linkage"/"nearest neighbor" method
1230 // should should be >
1231 // results seem to be pretty much the same with either method
1232
1233 // figure out which to delete and which to replace
1234 if ($sum1 > $sum2) {
1235 $replaceme = $highest_key1;
1236 $deleteme = $highest_key2;
1237 } else {
1238 $replaceme = $highest_key2;
1239 $deleteme = $highest_key1;
1240 }
1241
1242 $newkey = $replaceme . ':' . $deleteme;
1243
1244 // $replaceme is most similar to remaining languages
1245 // replace $replaceme with '$newkey', deleting $deleteme
1246
1247 // keep a record of which fork is really which language
1248 $really_lang = $replaceme;
1249 while (isset($really_map[$really_lang])) {
1250 $really_lang = $really_map[$really_lang];
1251 }
1252 $really_map[$newkey] = $really_lang;
1253
1254
1255 // replace the best fitting key, delete the other
1256 foreach ($arr as $key1 => $arr2) {
1257 foreach ($arr2 as $key2 => $value2) {
1258 if ($key2 == $replaceme) {
1259 $arr[$key1][$newkey] = $arr[$key1][$key2];
1260 unset($arr[$key1][$key2]);
1261 // replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
1262 }
1263
1264 if ($key1 == $replaceme) {
1265 $arr[$newkey][$key2] = $arr[$key1][$key2];
1266 unset($arr[$key1][$key2]);
1267 // replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
1268 }
1269
1270 if ($key1 == $deleteme || $key2 == $deleteme) {
1271 // deleting $arr[$key1][$key2]
1272 unset($arr[$key1][$key2]);
1273 }
1274 }
1275 }
1276
1277
1278 unset($langs[$highest_key1]);
1279 unset($langs[$highest_key2]);
1280 $langs[$newkey] = $newkey;
1281
1282
1283 // some of these may be overkill
1284 $result_data[$newkey] = array(
1285 'newkey' => $newkey,
1286 'count' => $i,
1287 'diff' => abs($sum1 - $sum2),
1288 'score' => $highest_score,
1289 'bestfit' => $replaceme,
1290 'otherfit' => $deleteme,
1291 'really' => $really_lang,
1292 );
1293 }
1294
1295 $return_val = array(
1296 'open_forks' => $langs,
1297 // the top level of clusters
1298 // clusters that are mutually exclusive
1299 // or specified by a specific maximum
1300
1301 'fork_data' => $result_data,
1302 // data for each split
1303
1304 'name_map' => $really_map,
1305 // which cluster is really which language
1306 // using the nearest neighbor technique, the cluster
1307 // inherits all of the properties of its most-similar member
1308 // this keeps track
1309 );
1310
1311
1312 // saves the result in the object
1313 $this->_clusters = $return_val;
1314
1315 return $return_val;
1316 }
1317
1318
1319 /**
1320 * Perform an intelligent detection based on clusterLanguages()
1321 *
1322 * WARNING: this method is EXPERIMENTAL. It is not recommended for common
1323 * use, and it may disappear or its functionality may change in future
1324 * releases without notice.
1325 *
1326 * This compares the sample text to top the top level of clusters. If the
1327 * sample is similar to the cluster it will drop down and compare it to the
1328 * languages in the cluster, and so on until it hits a leaf node.
1329 *
1330 * this should find the language in considerably fewer compares
1331 * (the equivalent of a binary search), however clusterLanguages() is costly
1332 * and the loss of accuracy from this technique is significant.
1333 *
1334 * This method may need to be 'fuzzier' in order to become more accurate.
1335 *
1336 * This function could be more useful if the universe of possible languages
1337 * was very large, however in such cases some method of Bayesian inference
1338 * might be more helpful.
1339 *
1340 * @see clusterLanguages()
1341 * @access public
1342 * @param string $str input string
1343 * @return array language scores (only those compared)
1344 * @throws PEAR_Error
1345 */
1346 function clusteredSearch($str)
1347 {
1348
1349 // input check
1350 if (!Text_LanguageDetect_Parser::validateString($str)) {
1351 return array();
1352 }
1353
1354 // clusterLanguages() will return a cached result if possible
1355 // so it's safe to call it every time
1356 $result = $this->clusterLanguages();
1357
1358 $dendogram_start = $result['open_forks'];
1359 $dendogram_data = $result['fork_data'];
1360 $dendogram_alias = $result['name_map'];
1361
1362 $sample_obj = new Text_LanguageDetect_Parser($str, $this->_db_filename, $this->_unicode_db_filename);
1363 $sample_obj->prepareTrigram();
1364 $sample_obj->setPadStart(!$this->_perl_compatible);
1365 $sample_obj->analyze();
1366 $sample_result = $sample_obj->getTrigramRanks();
1367 $sample_count = count($sample_result);
1368
1369 // input check
1370 if ($sample_count == 0) {
1371 return array();
1372 }
1373
1374 $i = 0; // counts the number of steps
1375
1376 foreach ($dendogram_start as $lang) {
1377 if (isset($dendogram_alias[$lang])) {
1378 $lang_key = $dendogram_alias[$lang];
1379 } else {
1380 $lang_key = $lang;
1381 }
1382
1383 $scores[$lang] = $this->_normalize_score(
1384 $this->_distance($this->_lang_db[$lang_key], $sample_result),
1385 $sample_count);
1386
1387 $i++;
1388 }
1389
1390 if ($this->_perl_compatible) {
1391 asort($scores);
1392 } else {
1393 arsort($scores);
1394 }
1395
1396 $top_score = current($scores);
1397 $top_key = key($scores);
1398
1399 // of starting forks, $top_key is the most similar to the sample
1400
1401 $cur_key = $top_key;
1402 while (isset($dendogram_data[$cur_key])) {
1403 $lang1 = $dendogram_data[$cur_key]['bestfit'];
1404 $lang2 = $dendogram_data[$cur_key]['otherfit'];
1405 foreach (array($lang1, $lang2) as $lang) {
1406 if (isset($dendogram_alias[$lang])) {
1407 $lang_key = $dendogram_alias[$lang];
1408 } else {
1409 $lang_key = $lang;
1410 }
1411
1412 $scores[$lang] = $this->_normalize_score(
1413 $this->_distance($this->_lang_db[$lang_key], $sample_result),
1414 $sample_count);
1415
1416 //todo: does not need to do same comparison again
1417 }
1418
1419 $i++;
1420
1421 if ($scores[$lang1] > $scores[$lang2]) {
1422 $cur_key = $lang1;
1423 $loser_key = $lang2;
1424 } else {
1425 $cur_key = $lang2;
1426 $loser_key = $lang1;
1427 }
1428
1429 $diff = $scores[$cur_key] - $scores[$loser_key];
1430
1431 // $cur_key ({$dendogram_alias[$cur_key]}) wins
1432 // over $loser_key ({$dendogram_alias[$loser_key]})
1433 // with a difference of $diff
1434 }
1435
1436 // found result in $i compares
1437
1438 // rather than sorting the result, preserve it so that you can see
1439 // which paths the algorithm decided to take along the tree
1440
1441 // but sometimes the last item is only the second highest
1442 if ( ($this->_perl_compatible && (end($scores) > prev($scores)))
1443 || (!$this->_perl_compatible && (end($scores) < prev($scores)))) {
1444
1445 $real_last_score = current($scores);
1446 $real_last_key = key($scores);
1447
1448 // swaps the 2nd-to-last item for the last item
1449 unset($scores[$real_last_key]);
1450 $scores[$real_last_key] = $real_last_score;
1451 }
1452
1453
1454 if (!$this->_perl_compatible) {
1455 $scores = array_reverse($scores, true);
1456 // second param requires php > 4.0.3
1457 }
1458
1459 return $scores;
1460 }
1461
1462 /**
1463 * ut8-safe strlen()
1464 *
1465 * Returns the numbers of characters (not bytes) in a utf8 string
1466 *
1467 * @static
1468 * @access public
1469 * @param string $str string to get the length of
1470 * @return int number of chars
1471 */
1472 function utf8strlen($str)
1473 {
1474 // utf8_decode() will convert unknown chars to '?', which is actually
1475 // ideal for counting.
1476
1477 return strlen(utf8_decode($str));
1478
1479 // idea stolen from dokuwiki
1480 }
1481
1482 /**
1483 * Returns the unicode value of a utf8 char
1484 *
1485 * @access protected
1486 * @param string $char a utf8 (possibly multi-byte) char
1487 * @return int unicode value or -1 if malformatted
1488 */
1489 function _utf8char2unicode($char) {
1490
1491 // strlen() here will actually get the binary length of a single char
1492 switch (strlen($char)) {
1493
1494 // for a reference, see http://en.wikipedia.org/wiki/UTF-8
1495
1496 case 1:
1497 // normal ASCII-7 byte
1498 // 0xxxxxxx --> 0xxxxxxx
1499 return ord($char{0});
1500
1501 case 2:
1502 // 2 byte unicode
1503 // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
1504 $z = (ord($char{0}) & 0x000001F) << 6;
1505 $x = (ord($char{1}) & 0x0000003F);
1506
1507 return ($z | $x);
1508
1509 case 3:
1510 // 3 byte unicode
1511 // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
1512 $z = (ord($char{0}) & 0x0000000F) << 12;
1513 $x1 = (ord($char{1}) & 0x0000003F) << 6;
1514 $x2 = (ord($char{2}) & 0x0000003F);
1515
1516 return ($z | $x1 | $x2);
1517
1518 case 4:
1519 // 4 byte unicode
1520 // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
1521 // 000zzzzz xxxxxxxx xxxxxxxx
1522 $z1 = (ord($char{0}) & 0x00000007) << 18;
1523 $z2 = (ord($char{1}) & 0x0000003F) << 12;
1524 $x1 = (ord($char{2}) & 0x0000003F) << 6;
1525 $x2 = (ord($char{3}) & 0x0000003F);
1526
1527 return ($z1 | $z2 | $x1 | $x2);
1528
1529 default:
1530 // error: malformatted char?
1531 return -1;
1532 }
1533 }
1534
1535 /**
1536 * utf8-safe fast character iterator
1537 *
1538 * Will get the next character starting from $counter, which will then be
1539 * incremented. If a multi-byte char the bytes will be concatenated and
1540 * $counter will be incremeted by the number of bytes in the char.
1541 *
1542 * @access private
1543 * @param string &$str the string being iterated over
1544 * @param int &$counter the iterator, will increment by reference
1545 * @param bool $special_convert whether to do special conversions
1546 * @return char the next (possibly multi-byte) char from $counter
1547 */
1548 function _next_char(&$str, &$counter, $special_convert = false)
1549 {
1550
1551 $char = $str{$counter++};
1552 $ord = ord($char);
1553
1554 // for a description of the utf8 system see
1555 // http://www.phpclasses.org/browse/file/5131.html
1556
1557 // normal ascii one byte char
1558 if ($ord <= 127) {
1559
1560 // special conversions needed for this package
1561 // (that only apply to regular ascii characters)
1562 // lower case, and convert all non-alphanumeric characters
1563 // other than "'" to space
1564 if ($special_convert && $char != ' ' && $char != "'") {
1565 if ($ord >= 65 && $ord <= 90) { // A-Z
1566 $char = chr($ord + 32); // lower case
1567 } elseif ($ord < 97 || $ord > 122) { // NOT a-z
1568 $char = ' '; // convert to space
1569 }
1570 }
1571
1572 return $char;
1573
1574 // multi-byte chars
1575 } elseif ($ord >> 5 == 6) { // two-byte char
1576 $nextchar = $str{$counter++}; // get next byte
1577
1578 // lower-casing of non-ascii characters is still incomplete
1579
1580 if ($special_convert) {
1581 // lower case latin accented characters
1582 if ($ord == 195) {
1583 $nextord = ord($nextchar);
1584 $nextord_adj = $nextord + 64;
1585 // for a reference, see
1586 // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
1587
1588 // &Agrave; - &THORN; but not &times;
1589 if ( $nextord_adj >= 192
1590 && $nextord_adj <= 222
1591 && $nextord_adj != 215) {
1592
1593 $nextchar = chr($nextord + 32);
1594 }
1595
1596 // lower case cyrillic alphabet
1597 } elseif ($ord == 208) {
1598 $nextord = ord($nextchar);
1599 // if A - Pe
1600 if ($nextord >= 144 && $nextord <= 159) {
1601 // lower case
1602 $nextchar = chr($nextord + 32);
1603
1604 // if Er - Ya
1605 } elseif ($nextord >= 160 && $nextord <= 175) {
1606 // lower case
1607 $char = chr(209); // == $ord++
1608 $nextchar = chr($nextord - 32);
1609 }
1610 }
1611 }
1612
1613 // tag on next byte
1614 return $char . $nextchar;
1615
1616 } elseif ($ord >> 4 == 14) { // three-byte char
1617
1618 // tag on next 2 bytes
1619 return $char . $str{$counter++} . $str{$counter++};
1620
1621 } elseif ($ord >> 3 == 30) { // four-byte char
1622
1623 // tag on next 3 bytes
1624 return $char . $str{$counter++} . $str{$counter++} . $str{$counter++};
1625
1626 } else {
1627 // error?
1628 }
1629 }
1630
1631}
1632
1633/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
1634
1635?>
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 05:59:44 +0000