vendor/twig/twig/lib/Twig/Lexer.php

   1 <?php
   2
   3 /*
   4  * This file is part of Twig.
   5  *
   6  * (c) 2009 Fabien Potencier
   7  * (c) 2009 Armin Ronacher
   8  *
   9  * For the full copyright and license information, please view the LICENSE
  10  * file that was distributed with this source code.
  11  */
  12
  13 /**
  14  * Lexes a template string.
  15  *
  16  * @author Fabien Potencier <fabien@symfony.com>
  17  */
  18 class Twig_Lexer implements Twig_LexerInterface
  19 {
  20     protected $tokens;
  21     protected $code;
  22     protected $cursor;
  23     protected $lineno;
  24     protected $end;
  25     protected $state;
  26     protected $states;
  27     protected $brackets;
  28     protected $env;
  29     protected $filename;
  30     protected $options;
  31     protected $regexes;
  32     protected $position;
  33     protected $positions;
  34     protected $currentVarBlockLine;
  35
  36     const STATE_DATA            = 0;
  37     const STATE_BLOCK           = 1;
  38     const STATE_VAR             = 2;
  39     const STATE_STRING          = 3;
  40     const STATE_INTERPOLATION   = 4;
  41
  42     const REGEX_NAME            = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
  43     const REGEX_NUMBER          = '/[0-9]+(?:\.[0-9]+)?/A';
  44     const REGEX_STRING          = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
  45     const REGEX_DQ_STRING_DELIM = '/"/A';
  46     const REGEX_DQ_STRING_PART  = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
  47     const PUNCTUATION           = '()[]{}?:.,|';
  48
  49     public function __construct(Twig_Environment $env, array $options = array())
  50     {
  51         $this->env = $env;
  52
  53         $this->options = array_merge(array(
  54             'tag_comment'     => array('{#', '#}'),
  55             'tag_block'       => array('{%', '%}'),
  56             'tag_variable'    => array('{{', '}}'),
  57             'whitespace_trim' => '-',
  58             'interpolation'   => array('#{', '}'),
  59         ), $options);
  60
  61         $this->regexes = array(
  62             'lex_var'             => '/\s*'.preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_variable'][1], '/').'/A',
  63             'lex_block'           => '/\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')\n?/A',
  64             'lex_raw_data'        => '/('.preg_quote($this->options['tag_block'][0].$this->options['whitespace_trim'], '/').'|'.preg_quote($this->options['tag_block'][0], '/').')\s*(?:end%s)\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')/s',
  65             'operator'            => $this->getOperatorRegex(),
  66             'lex_comment'         => '/(?:'.preg_quote($this->options['whitespace_trim'], '/').preg_quote($this->options['tag_comment'][1], '/').'\s*|'.preg_quote($this->options['tag_comment'][1], '/').')\n?/s',
  67             'lex_block_raw'       => '/\s*(raw|verbatim)\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')/As',
  68             'lex_block_line'      => '/\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '/').'/As',
  69             'lex_tokens_start'    => '/('.preg_quote($this->options['tag_variable'][0], '/').'|'.preg_quote($this->options['tag_block'][0], '/').'|'.preg_quote($this->options['tag_comment'][0], '/').')('.preg_quote($this->options['whitespace_trim'], '/').')?/s',
  70             'interpolation_start' => '/'.preg_quote($this->options['interpolation'][0], '/').'\s*/A',
  71             'interpolation_end'   => '/\s*'.preg_quote($this->options['interpolation'][1], '/').'/A',
  72         );
  73     }
  74
  75     /**
  76      * Tokenizes a source code.
  77      *
  78      * @param string $code     The source code
  79      * @param string $filename A unique identifier for the source code
  80      *
  81      * @return Twig_TokenStream A token stream instance
  82      */
  83     public function tokenize($code, $filename = null)
  84     {
  85         if (function_exists('mb_internal_encoding') && ((int) ini_get('mbstring.func_overload')) & 2) {
  86             $mbEncoding = mb_internal_encoding();
  87             mb_internal_encoding('ASCII');
  88         }
  89
  90         $this->code = str_replace(array("\r\n", "\r"), "\n", $code);
  91         $this->filename = $filename;
  92         $this->cursor = 0;
  93         $this->lineno = 1;
  94         $this->end = strlen($this->code);
  95         $this->tokens = array();
  96         $this->state = self::STATE_DATA;
  97         $this->states = array();
  98         $this->brackets = array();
  99         $this->position = -1;
 100
 101         // find all token starts in one go
 102         preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, PREG_OFFSET_CAPTURE);
 103         $this->positions = $matches;
 104
 105         while ($this->cursor < $this->end) {
 106             // dispatch to the lexing functions depending
 107             // on the current state
 108             switch ($this->state) {
 109                 case self::STATE_DATA:
 110                     $this->lexData();
 111                     break;
 112
 113                 case self::STATE_BLOCK:
 114                     $this->lexBlock();
 115                     break;
 116
 117                 case self::STATE_VAR:
 118                     $this->lexVar();
 119                     break;
 120
 121                 case self::STATE_STRING:
 122                     $this->lexString();
 123                     break;
 124
 125                 case self::STATE_INTERPOLATION:
 126                     $this->lexInterpolation();
 127                     break;
 128             }
 129         }
 130
 131         $this->pushToken(Twig_Token::EOF_TYPE);
 132
 133         if (!empty($this->brackets)) {
 134             list($expect, $lineno) = array_pop($this->brackets);
 135             throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
 136         }
 137
 138         if (isset($mbEncoding)) {
 139             mb_internal_encoding($mbEncoding);
 140         }
 141
 142         return new Twig_TokenStream($this->tokens, $this->filename);
 143     }
 144
 145     protected function lexData()
 146     {
 147         // if no matches are left we return the rest of the template as simple text token
 148         if ($this->position == count($this->positions[0]) - 1) {
 149             $this->pushToken(Twig_Token::TEXT_TYPE, substr($this->code, $this->cursor));
 150             $this->cursor = $this->end;
 151
 152             return;
 153         }
 154
 155         // Find the first token after the current cursor
 156         $position = $this->positions[0][++$this->position];
 157         while ($position[1] < $this->cursor) {
 158             if ($this->position == count($this->positions[0]) - 1) {
 159                 return;
 160             }
 161             $position = $this->positions[0][++$this->position];
 162         }
 163
 164         // push the template text first
 165         $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
 166         if (isset($this->positions[2][$this->position][0])) {
 167             $text = rtrim($text);
 168         }
 169         $this->pushToken(Twig_Token::TEXT_TYPE, $text);
 170         $this->moveCursor($textContent.$position[0]);
 171
 172         switch ($this->positions[1][$this->position][0]) {
 173             case $this->options['tag_comment'][0]:
 174                 $this->lexComment();
 175                 break;
 176
 177             case $this->options['tag_block'][0]:
 178                 // raw data?
 179                 if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, null, $this->cursor)) {
 180                     $this->moveCursor($match[0]);
 181                     $this->lexRawData($match[1]);
 182                 // {% line \d+ %}
 183                 } elseif (preg_match($this->regexes['lex_block_line'], $this->code, $match, null, $this->cursor)) {
 184                     $this->moveCursor($match[0]);
 185                     $this->lineno = (int) $match[1];
 186                 } else {
 187                     $this->pushToken(Twig_Token::BLOCK_START_TYPE);
 188                     $this->pushState(self::STATE_BLOCK);
 189                     $this->currentVarBlockLine = $this->lineno;
 190                 }
 191                 break;
 192
 193             case $this->options['tag_variable'][0]:
 194                 $this->pushToken(Twig_Token::VAR_START_TYPE);
 195                 $this->pushState(self::STATE_VAR);
 196                 $this->currentVarBlockLine = $this->lineno;
 197                 break;
 198         }
 199     }
 200
 201     protected function lexBlock()
 202     {
 203         if (empty($this->brackets) && preg_match($this->regexes['lex_block'], $this->code, $match, null, $this->cursor)) {
 204             $this->pushToken(Twig_Token::BLOCK_END_TYPE);
 205             $this->moveCursor($match[0]);
 206             $this->popState();
 207         } else {
 208             $this->lexExpression();
 209         }
 210     }
 211
 212     protected function lexVar()
 213     {
 214         if (empty($this->brackets) && preg_match($this->regexes['lex_var'], $this->code, $match, null, $this->cursor)) {
 215             $this->pushToken(Twig_Token::VAR_END_TYPE);
 216             $this->moveCursor($match[0]);
 217             $this->popState();
 218         } else {
 219             $this->lexExpression();
 220         }
 221     }
 222
 223     protected function lexExpression()
 224     {
 225         // whitespace
 226         if (preg_match('/\s+/A', $this->code, $match, null, $this->cursor)) {
 227             $this->moveCursor($match[0]);
 228
 229             if ($this->cursor >= $this->end) {
 230                 throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $this->state === self::STATE_BLOCK ? 'block' : 'variable'), $this->currentVarBlockLine, $this->filename);
 231             }
 232         }
 233
 234         // operators
 235         if (preg_match($this->regexes['operator'], $this->code, $match, null, $this->cursor)) {
 236             $this->pushToken(Twig_Token::OPERATOR_TYPE, $match[0]);
 237             $this->moveCursor($match[0]);
 238         }
 239         // names
 240         elseif (preg_match(self::REGEX_NAME, $this->code, $match, null, $this->cursor)) {
 241             $this->pushToken(Twig_Token::NAME_TYPE, $match[0]);
 242             $this->moveCursor($match[0]);
 243         }
 244         // numbers
 245         elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, null, $this->cursor)) {
 246             $number = (float) $match[0];  // floats
 247             if (ctype_digit($match[0]) && $number <= PHP_INT_MAX) {
 248                 $number = (int) $match[0]; // integers lower than the maximum
 249             }
 250             $this->pushToken(Twig_Token::NUMBER_TYPE, $number);
 251             $this->moveCursor($match[0]);
 252         }
 253         // punctuation
 254         elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
 255             // opening bracket
 256             if (false !== strpos('([{', $this->code[$this->cursor])) {
 257                 $this->brackets[] = array($this->code[$this->cursor], $this->lineno);
 258             }
 259             // closing bracket
 260             elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
 261                 if (empty($this->brackets)) {
 262                     throw new Twig_Error_Syntax(sprintf('Unexpected "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
 263                 }
 264
 265                 list($expect, $lineno) = array_pop($this->brackets);
 266                 if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
 267                     throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
 268                 }
 269             }
 270
 271             $this->pushToken(Twig_Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
 272             ++$this->cursor;
 273         }
 274         // strings
 275         elseif (preg_match(self::REGEX_STRING, $this->code, $match, null, $this->cursor)) {
 276             $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes(substr($match[0], 1, -1)));
 277             $this->moveCursor($match[0]);
 278         }
 279         // opening double quoted string
 280         elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, null, $this->cursor)) {
 281             $this->brackets[] = array('"', $this->lineno);
 282             $this->pushState(self::STATE_STRING);
 283             $this->moveCursor($match[0]);
 284         }
 285         // unlexable
 286         else {
 287             throw new Twig_Error_Syntax(sprintf('Unexpected character "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
 288         }
 289     }
 290
 291     protected function lexRawData($tag)
 292     {
 293         if (!preg_match(str_replace('%s', $tag, $this->regexes['lex_raw_data']), $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
 294             throw new Twig_Error_Syntax(sprintf('Unexpected end of file: Unclosed "%s" block', $tag), $this->lineno, $this->filename);
 295         }
 296
 297         $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
 298         $this->moveCursor($text.$match[0][0]);
 299
 300         if (false !== strpos($match[1][0], $this->options['whitespace_trim'])) {
 301             $text = rtrim($text);
 302         }
 303
 304         $this->pushToken(Twig_Token::TEXT_TYPE, $text);
 305     }
 306
 307     protected function lexComment()
 308     {
 309         if (!preg_match($this->regexes['lex_comment'], $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
 310             throw new Twig_Error_Syntax('Unclosed comment', $this->lineno, $this->filename);
 311         }
 312
 313         $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
 314     }
 315
 316     protected function lexString()
 317     {
 318         if (preg_match($this->regexes['interpolation_start'], $this->code, $match, null, $this->cursor)) {
 319             $this->brackets[] = array($this->options['interpolation'][0], $this->lineno);
 320             $this->pushToken(Twig_Token::INTERPOLATION_START_TYPE);
 321             $this->moveCursor($match[0]);
 322             $this->pushState(self::STATE_INTERPOLATION);
 323
 324         } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, null, $this->cursor) && strlen($match[0]) > 0) {
 325             $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes($match[0]));
 326             $this->moveCursor($match[0]);
 327
 328         } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, null, $this->cursor)) {
 329
 330             list($expect, $lineno) = array_pop($this->brackets);
 331             if ($this->code[$this->cursor] != '"') {
 332                 throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
 333             }
 334
 335             $this->popState();
 336             ++$this->cursor;
 337         }
 338     }
 339
 340     protected function lexInterpolation()
 341     {
 342         $bracket = end($this->brackets);
 343         if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, null, $this->cursor)) {
 344             array_pop($this->brackets);
 345             $this->pushToken(Twig_Token::INTERPOLATION_END_TYPE);
 346             $this->moveCursor($match[0]);
 347             $this->popState();
 348         } else {
 349             $this->lexExpression();
 350         }
 351     }
 352
 353     protected function pushToken($type, $value = '')
 354     {
 355         // do not push empty text tokens
 356         if (Twig_Token::TEXT_TYPE === $type && '' === $value) {
 357             return;
 358         }
 359
 360         $this->tokens[] = new Twig_Token($type, $value, $this->lineno);
 361     }
 362
 363     protected function moveCursor($text)
 364     {
 365         $this->cursor += strlen($text);
 366         $this->lineno += substr_count($text, "\n");
 367     }
 368
 369     protected function getOperatorRegex()
 370     {
 371         $operators = array_merge(
 372             array('='),
 373             array_keys($this->env->getUnaryOperators()),
 374             array_keys($this->env->getBinaryOperators())
 375         );
 376
 377         $operators = array_combine($operators, array_map('strlen', $operators));
 378         arsort($operators);
 379
 380         $regex = array();
 381         foreach ($operators as $operator => $length) {
 382             // an operator that ends with a character must be followed by
 383             // a whitespace or a parenthesis
 384             if (ctype_alpha($operator[$length - 1])) {
 385                 $regex[] = preg_quote($operator, '/').'(?=[\s()])';
 386             } else {
 387                 $regex[] = preg_quote($operator, '/');
 388             }
 389         }
 390
 391         return '/'.implode('|', $regex).'/A';
 392     }
 393
 394     protected function pushState($state)
 395     {
 396         $this->states[] = $this->state;
 397         $this->state = $state;
 398     }
 399
 400     protected function popState()
 401     {
 402         if (0 === count($this->states)) {
 403             throw new Exception('Cannot pop state without a previous state');
 404         }
 405
 406         $this->state = array_pop($this->states);
 407     }
 408 }