JSTokenizer

The W3 Total Cache JSTokenizer class.

Defined (1)

The class is defined in the following location(s).

/lib/Minify/JSMinPlus.php  
  1. class JSTokenizer 
  2. { 
  3.     private $cursor = 0; 
  4.     private $source; 
  5.  
  6.     public $tokens = array(); 
  7.     public $tokenIndex = 0; 
  8.     public $lookahead = 0; 
  9.     public $scanNewlines = false; 
  10.     public $scanOperand = true; 
  11.  
  12.     public $filename; 
  13.     public $lineno; 
  14.  
  15.     private $keywords = array( 
  16.         'break',  
  17.         'case', 'catch', 'const', 'continue',  
  18.         'debugger', 'default', 'delete', 'do',  
  19.         'else', 'enum',  
  20.         'false', 'finally', 'for', 'function',  
  21.         'if', 'in', 'instanceof',  
  22.         'new', 'null',  
  23.         'return',  
  24.         'switch',  
  25.         'this', 'throw', 'true', 'try', 'typeof',  
  26.         'var', 'void',  
  27.         'while', 'with' 
  28.  ); 
  29.  
  30.     private $opTypeNames = array( 
  31.         ';', ', ', '?', ':', '||', '&&', '|', '^',  
  32.         '&', '===', '==', '=', '!==', '!=', '<<', '<=',  
  33.         '<', '>>>', '>>', '>=', '>', '++', '--', '+',  
  34.         '-', '*', '/', '%', '!', '~', '.', '[',  
  35.         ']', '{', '}', '(', ')', '@*/' 
  36.  ); 
  37.  
  38.     private $assignOps = array('|', '^', '&', '<<', '>>', '>>>', '+', '-', '*', '/', '%'); 
  39.     private $opRegExp; 
  40.  
  41.     public function __construct() 
  42.     { 
  43.         $this->opRegExp = '#^(' . implode('|', array_map('preg_quote', $this->opTypeNames)) . ')#'; 
  44.     } 
  45.  
  46.     public function init($source, $filename = '', $lineno = 1) 
  47.     { 
  48.         $this->source = $source; 
  49.         $this->filename = $filename ? $filename : '[inline]'; 
  50.         $this->lineno = $lineno; 
  51.  
  52.         $this->cursor = 0; 
  53.         $this->tokens = array(); 
  54.         $this->tokenIndex = 0; 
  55.         $this->lookahead = 0; 
  56.         $this->scanNewlines = false; 
  57.         $this->scanOperand = true; 
  58.     } 
  59.  
  60.     public function getInput($chunksize) 
  61.     { 
  62.         if ($chunksize) 
  63.             return substr($this->source, $this->cursor, $chunksize); 
  64.  
  65.         return substr($this->source, $this->cursor); 
  66.     } 
  67.  
  68.     public function isDone() 
  69.     { 
  70.         return $this->peek() == TOKEN_END; 
  71.     } 
  72.  
  73.     public function match($tt) 
  74.     { 
  75.         return $this->get() == $tt || $this->unget(); 
  76.     } 
  77.  
  78.     public function mustMatch($tt) 
  79.     { 
  80.             if (!$this->match($tt)) 
  81.             throw $this->newSyntaxError('Unexpected token; token ' . $tt . ' expected'); 
  82.  
  83.         return $this->currentToken(); 
  84.     } 
  85.  
  86.     public function peek() 
  87.     { 
  88.         if ($this->lookahead) 
  89.         { 
  90.             $next = $this->tokens[($this->tokenIndex + $this->lookahead) & 3]; 
  91.             if ($this->scanNewlines && $next->lineno != $this->lineno) 
  92.                 $tt = TOKEN_NEWLINE; 
  93.             else 
  94.                 $tt = $next->type; 
  95.         } 
  96.         else 
  97.         { 
  98.             $tt = $this->get(); 
  99.             $this->unget(); 
  100.         } 
  101.  
  102.         return $tt; 
  103.     } 
  104.  
  105.     public function peekOnSameLine() 
  106.     { 
  107.         $this->scanNewlines = true; 
  108.         $tt = $this->peek(); 
  109.         $this->scanNewlines = false; 
  110.  
  111.         return $tt; 
  112.     } 
  113.  
  114.     public function currentToken() 
  115.     { 
  116.         if (!empty($this->tokens)) 
  117.             return $this->tokens[$this->tokenIndex]; 
  118.     } 
  119.  
  120.     public function get($chunksize = 1000) 
  121.     { 
  122.         while($this->lookahead) 
  123.         { 
  124.             $this->lookahead--; 
  125.             $this->tokenIndex = ($this->tokenIndex + 1) & 3; 
  126.             $token = $this->tokens[$this->tokenIndex]; 
  127.             if ($token->type != TOKEN_NEWLINE || $this->scanNewlines) 
  128.                 return $token->type; 
  129.         } 
  130.  
  131.         $conditional_comment = false; 
  132.  
  133.         // strip whitespace and comments 
  134.         while(true) 
  135.         { 
  136.             $input = $this->getInput($chunksize); 
  137.  
  138.             // whitespace handling; gobble up \r as well (effectively we don't have support for MAC newlines!) 
  139.             $re = $this->scanNewlines ? '/^[ \r\t]+/' : '/^\s+/'; 
  140.             if (preg_match($re, $input, $match)) 
  141.             { 
  142.                 $spaces = $match[0]; 
  143.                 $spacelen = strlen($spaces); 
  144.                 $this->cursor += $spacelen; 
  145.                 if (!$this->scanNewlines) 
  146.                     $this->lineno += substr_count($spaces, "\n"); 
  147.  
  148.                 if ($spacelen == $chunksize) 
  149.                     continue; // complete chunk contained whitespace 
  150.  
  151.                 $input = $this->getInput($chunksize); 
  152.                 if ($input == '' || $input[0] != '/') 
  153.                     break; 
  154.             } 
  155.  
  156.             // Comments 
  157.             if (!preg_match('/^\/(?:\*(@(?:cc_on|if|elif|else|end))?.*?\*\/|\/[^\n]*)/s', $input, $match)) 
  158.             { 
  159.                 if (!$chunksize) 
  160.                     break; 
  161.  
  162.                 // retry with a full chunk fetch; this also prevents breakage of long regular expressions (which will never match a comment) 
  163.                 $chunksize = null; 
  164.                 continue; 
  165.             } 
  166.  
  167.             // check if this is a conditional (JScript) comment 
  168.             if (!empty($match[1])) 
  169.             { 
  170.                 $match[0] = '/*' . $match[1]; 
  171.                 $conditional_comment = true; 
  172.                 break; 
  173.             } 
  174.             else 
  175.             { 
  176.                 $this->cursor += strlen($match[0]); 
  177.                 $this->lineno += substr_count($match[0], "\n"); 
  178.             } 
  179.         } 
  180.   
  181.         if ($input == '') 
  182.         { 
  183.             $tt = TOKEN_END; 
  184.             $match = array(''); 
  185.         } 
  186.         elseif ($conditional_comment) 
  187.         { 
  188.             $tt = TOKEN_CONDCOMMENT_START; 
  189.         } 
  190.         else 
  191.         { 
  192.             switch ($input[0]) 
  193.             { 
  194.                 case '0': 
  195.                     // hexadecimal 
  196.                     if (($input[1] == 'x' || $input[1] == 'X') && preg_match('/^0x[0-9a-f]+/i', $input, $match)) 
  197.                     { 
  198.                         $tt = TOKEN_NUMBER; 
  199.                         break; 
  200.                     } 
  201.                 // FALL THROUGH 
  202.   
  203.                 case '1': case '2': case '3': case '4': case '5': 
  204.                 case '6': case '7': case '8': case '9': 
  205.                     // should always match 
  206.                     preg_match('/^\d+(?:\.\d*)?(?:[eE][-+]?\d+)?/', $input, $match); 
  207.                     $tt = TOKEN_NUMBER; 
  208.                 break; 
  209.   
  210.                 case "'": 
  211.                     if (preg_match('/^\'(?:[^\\\\\'\r\n]++|\\\\(?:.|\r?\n))*\'/', $input, $match)) 
  212.                     { 
  213.                         $tt = TOKEN_STRING; 
  214.                     } 
  215.                     else 
  216.                     { 
  217.                         if ($chunksize) 
  218.                             return $this->get(null); // retry with a full chunk fetch 
  219.   
  220.                         throw $this->newSyntaxError('Unterminated string literal'); 
  221.                     } 
  222.                 break; 
  223.   
  224.                 case '"': 
  225.                     if (preg_match('/^"(?:[^\\\\"\r\n]++|\\\\(?:.|\r?\n))*"/', $input, $match)) 
  226.                     { 
  227.                         $tt = TOKEN_STRING; 
  228.                     } 
  229.                     else 
  230.                     { 
  231.                         if ($chunksize) 
  232.                             return $this->get(null); // retry with a full chunk fetch 
  233.   
  234.                         throw $this->newSyntaxError('Unterminated string literal'); 
  235.                     } 
  236.                 break; 
  237.   
  238.                 case '/': 
  239.                     if ($this->scanOperand && preg_match('/^\/((?:\\\\.|\[(?:\\\\.|[^\]])*\]|[^\/])+)\/([gimy]*)/', $input, $match)) 
  240.                     { 
  241.                         $tt = TOKEN_REGEXP; 
  242.                         break; 
  243.                     } 
  244.                 // FALL THROUGH 
  245.   
  246.                 case '|': 
  247.                 case '^': 
  248.                 case '&': 
  249.                 case '<': 
  250.                 case '>': 
  251.                 case '+': 
  252.                 case '-': 
  253.                 case '*': 
  254.                 case '%': 
  255.                 case '=': 
  256.                 case '!': 
  257.                     // should always match 
  258.                     preg_match($this->opRegExp, $input, $match); 
  259.                     $op = $match[0]; 
  260.                     if (in_array($op, $this->assignOps) && $input[strlen($op)] == '=') 
  261.                     { 
  262.                         $tt = OP_ASSIGN; 
  263.                         $match[0] .= '='; 
  264.                     } 
  265.                     else 
  266.                     { 
  267.                         $tt = $op; 
  268.                         if ($this->scanOperand) 
  269.                         { 
  270.                             if ($op == OP_PLUS) 
  271.                                 $tt = OP_UNARY_PLUS; 
  272.                             elseif ($op == OP_MINUS) 
  273.                                 $tt = OP_UNARY_MINUS; 
  274.                         } 
  275.                         $op = null; 
  276.                     } 
  277.                 break; 
  278.   
  279.                 case '.': 
  280.                     if (preg_match('/^\.\d+(?:[eE][-+]?\d+)?/', $input, $match)) 
  281.                     { 
  282.                         $tt = TOKEN_NUMBER; 
  283.                         break; 
  284.                     } 
  285.                 // FALL THROUGH 
  286.   
  287.                 case ';': 
  288.                 case ', ': 
  289.                 case '?': 
  290.                 case ':': 
  291.                 case '~': 
  292.                 case '[': 
  293.                 case ']': 
  294.                 case '{': 
  295.                 case '}': 
  296.                 case '(': 
  297.                 case ')': 
  298.                     // these are all single 
  299.                     $match = array($input[0]); 
  300.                     $tt = $input[0]; 
  301.                 break; 
  302.   
  303.                 case '@': 
  304.                     // check end of conditional comment 
  305.                     if (substr($input, 0, 3) == '@*/') 
  306.                     { 
  307.                         $match = array('@*/'); 
  308.                         $tt = TOKEN_CONDCOMMENT_END; 
  309.                     } 
  310.                     else 
  311.                         throw $this->newSyntaxError('Illegal token'); 
  312.                 break; 
  313.  
  314.                 case "\n": 
  315.                     if ($this->scanNewlines) 
  316.                     { 
  317.                         $match = array("\n"); 
  318.                         $tt = TOKEN_NEWLINE; 
  319.                     } 
  320.                     else 
  321.                         throw $this->newSyntaxError('Illegal token'); 
  322.                 break; 
  323.  
  324.                 default: 
  325.                     // FIXME: add support for unicode and unicode escape sequence \uHHHH 
  326.                     if (preg_match('/^[$\w]+/', $input, $match)) 
  327.                     { 
  328.                         $tt = in_array($match[0], $this->keywords) ? $match[0] : TOKEN_IDENTIFIER; 
  329.                     } 
  330.                     else 
  331.                         throw $this->newSyntaxError('Illegal token'); 
  332.             } 
  333.         } 
  334.  
  335.         $this->tokenIndex = ($this->tokenIndex + 1) & 3; 
  336.  
  337.         if (!isset($this->tokens[$this->tokenIndex])) 
  338.             $this->tokens[$this->tokenIndex] = new JSToken(); 
  339.  
  340.         $token = $this->tokens[$this->tokenIndex]; 
  341.         $token->type = $tt; 
  342.  
  343.         if ($tt == OP_ASSIGN) 
  344.             $token->assignOp = $op; 
  345.  
  346.         $token->start = $this->cursor; 
  347.  
  348.         $token->value = $match[0]; 
  349.         $this->cursor += strlen($match[0]); 
  350.  
  351.         $token->end = $this->cursor; 
  352.         $token->lineno = $this->lineno; 
  353.  
  354.         return $tt; 
  355.     } 
  356.  
  357.     public function unget() 
  358.     { 
  359.         if (++$this->lookahead == 4) 
  360.             throw $this->newSyntaxError('PANIC: too much lookahead!'); 
  361.  
  362.         $this->tokenIndex = ($this->tokenIndex - 1) & 3; 
  363.     } 
  364.  
  365.     public function newSyntaxError($m) 
  366.     { 
  367.         return new Exception('Parse error: ' . $m . ' in file \'' . $this->filename . '\' on line ' . $this->lineno); 
  368.     } 
  369. }