4 * This class represents a text sample to be parsed.
7 * @package Text_LanguageDetect
8 * @author Nicholas Pisarro
11 * @version CVS: $Id: Parser.php,v 1.5 2006/03/11 05:45:05 taak Exp $
12 * @link http://pear.php.net/package/Text_LanguageDetect/
13 * @link http://langdetect.blogspot.com/
17 * This class represents a text sample to be parsed.
19 * This separates the analysis of a text sample from the primary LanguageDetect
20 * class. After a new profile has been built, the data can be retrieved using
21 * the accessor functions.
23 * This class is intended to be used by the Text_LanguageDetect class, not
27 * @package Text_LanguageDetect
28 * @author Nicholas Pisarro
31 * @version release: 0.2.3
33 class Text_LanguageDetect_Parser
extends Text_LanguageDetect
36 * the piece of text being parsed
44 * stores the trigram frequencies of the sample
49 var $_trigrams = array();
52 * stores the trigram ranks of the sample
57 var $_trigram_ranks = array();
60 * stores the unicode blocks of the sample
65 var $_unicode_blocks = array();
68 * Whether the parser should compile the unicode ranges
73 var $_compile_unicode = false;
76 * Whether the parser should compile trigrams
81 var $_compile_trigram = false;
84 * Whether the trigram parser should pad the beginning of the string
89 var $_trigram_pad_start = false;
92 * Whether the unicode parser should skip non-alphabetical ascii chars
97 var $_unicode_skip_symbols = true;
103 * @param string $string string to be parsed
105 function Text_LanguageDetect_Parser($string, $db=null, $unicode_db=null) {
106 if (isset($db)) $this->_db_filename
= $db;
107 if (isset($unicode_db)) $this->_unicode_db_filename
= $unicode_db;
108 $this->_string
= $string;
112 * Returns true if a string is suitable for parsing
116 * @param string $str input string to test
117 * @return bool true if acceptable, false if not
119 function validateString($str) {
120 if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) {
128 * turn on/off trigram counting
131 * @param bool $bool true for on, false for off
133 function prepareTrigram($bool = true)
135 $this->_compile_trigram
= $bool;
139 * turn on/off unicode block counting
142 * @param bool $bool true for on, false for off
144 function prepareUnicode($bool = true)
146 $this->_compile_unicode
= $bool;
150 * turn on/off padding the beginning of the sample string
153 * @param bool $bool true for on, false for off
155 function setPadStart($bool = true)
157 $this->_trigram_pad_start
= $bool;
161 * Should the unicode block counter skip non-alphabetical ascii chars?
164 * @param bool $bool true for on, false for off
166 function setUnicodeSkipSymbols($bool = true)
168 $this->_unicode_skip_symbols
= $bool;
172 * Returns the trigram ranks for the text sample
175 * @return array trigram ranks in the text sample
177 function &getTrigramRanks()
179 return $this->_trigram_ranks
;
183 * Return the trigram freqency table
185 * only used in testing to make sure the parser is working
188 * @return array trigram freqencies in the text sample
190 function &getTrigramFreqs()
192 return $this->_trigram
;
196 * returns the array of unicode blocks
199 * @return array unicode blocks in the text sample
201 function &getUnicodeBlocks()
203 return $this->_unicode_blocks
;
207 * Executes the parsing operation
209 * Be sure to call the set*() functions to set options and the
210 * prepare*() functions first to tell it what kind of data to compute
212 * Afterwards the get*() functions can be used to access the compiled
219 $len = strlen($this->_string
);
224 if ($this->_compile_unicode
) {
225 $blocks =& $this->_read_unicode_block_db();
227 $block_count = count($blocks);
230 $unicode_chars = array();
234 if ($this->_compile_trigram
) {
235 // initialize them as blank so the parser will skip the first two
236 // (since it skips trigrams with more than 2 contiguous spaces)
241 // if it finds a valid trigram to start and the start pad option is
242 // off, then set a variable that will be used to reduce this
243 // trigram after parsing has finished
244 if (!$this->_trigram_pad_start
) {
245 $a = $this->_next_char($this->_string
, $byte_counter, true);
248 $b = $this->_next_char($this->_string
, $byte_counter, true);
258 while ($byte_counter < $len) {
259 $char = $this->_next_char($this->_string, $byte_counter, true);
262 // language trigram detection
263 if ($this->_compile_trigram) {
264 if (!($b == ' ' && ($a == ' ' || $char == ' '))) {
265 if (!isset($this->_trigram[$a . $b . $char])) {
266 $this->_trigram[$a . $b . $char] = 1;
268 $this->_trigram[$a . $b . $char]++;
276 // unicode block detection
277 if ($this->_compile_unicode) {
278 if ($this->_unicode_skip_symbols
279 && strlen($char) == 1
280 && ($char < 'A' || $char > 'z'
281 || ($char > 'Z' && $char < 'a'))
282 && $char != "'") { // does not skip the apostrophe
283 // since it's included in the language
290 // build an array of all the characters
291 if (isset($unicode_chars[$char])) {
292 $unicode_chars[$char]++
;
294 $unicode_chars[$char] = 1;
298 // todo: add byte detection here
302 if ($this->_compile_unicode
) {
303 foreach ($unicode_chars as $utf8_char => $count) {
304 $search_result = $this->_unicode_block_name(
305 $this->_utf8char2unicode($utf8_char), $blocks, $block_count);
307 if ($search_result != -1) {
308 $block_name = $search_result[2];
310 $block_name = '[Malformatted]';
313 if (isset($this->_unicode_blocks
[$block_name])) {
314 $this->_unicode_blocks
[$block_name] +
= $count;
316 $this->_unicode_blocks
[$block_name] = $count;
323 if ($this->_compile_trigram
) {
326 if (!isset($this->_trigram
["$a$b "])) {
327 $this->_trigram
["$a$b "] = 1;
329 $this->_trigram
["$a$b "]++
;
333 // perl compatibility; Language::Guess does not pad the beginning
335 if (isset($dropone)) {
336 if ($this->_trigram
[$dropone] == 1) {
337 unset($this->_trigram
[$dropone]);
339 $this->_trigram
[$dropone]--;
343 if (!empty($this->_trigram
)) {
344 $this->_trigram_ranks
= $this->_arr_rank($this->_trigram
);
346 $this->_trigram_ranks
= array();
352 /* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */