| Current File : /var/www/prestashop/modules/ps_metrics/vendor/squizlabs/php_codesniffer/src/Tokenizers/JS.php |
<?php
/**
* Tokenizes JS code.
*
* @author Greg Sherwood <gsherwood@squiz.net>
* @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
*/
namespace ps_metrics_module_v4_0_6\PHP_CodeSniffer\Tokenizers;
use ps_metrics_module_v4_0_6\PHP_CodeSniffer\Config;
use ps_metrics_module_v4_0_6\PHP_CodeSniffer\Exceptions\TokenizerException;
use ps_metrics_module_v4_0_6\PHP_CodeSniffer\Util;
class JS extends Tokenizer
{
/**
* A list of tokens that are allowed to open a scope.
*
* This array also contains information about what kind of token the scope
* opener uses to open and close the scope, if the token strictly requires
* an opener, if the token can share a scope closer, and who it can be shared
* with. An example of a token that shares a scope closer is a CASE scope.
*
* @var array
*/
public $scopeOpeners = [\T_IF => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \false, 'shared' => \false, 'with' => []], \T_TRY => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \true, 'shared' => \false, 'with' => []], \T_CATCH => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \true, 'shared' => \false, 'with' => []], \T_ELSE => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \false, 'shared' => \false, 'with' => []], \T_FOR => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \false, 'shared' => \false, 'with' => []], \T_CLASS => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \true, 'shared' => \false, 'with' => []], \T_FUNCTION => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \false, 'shared' => \false, 'with' => []], \T_WHILE => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \false, 'shared' => \false, 'with' => []], \T_DO => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \true, 'shared' => \false, 'with' => []], \T_SWITCH => ['start' => [T_OPEN_CURLY_BRACKET => T_OPEN_CURLY_BRACKET], 'end' => [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET], 'strict' => \true, 'shared' => \false, 'with' => []], \T_CASE => ['start' => [T_COLON => T_COLON], 'end' => [\T_BREAK => \T_BREAK, \T_RETURN => \T_RETURN, \T_CONTINUE => \T_CONTINUE, \T_THROW => \T_THROW], 'strict' => \true, 'shared' => \true, 'with' => [\T_DEFAULT => \T_DEFAULT, \T_CASE => \T_CASE, \T_SWITCH => \T_SWITCH]], \T_DEFAULT => ['start' => [T_COLON => T_COLON], 'end' => [\T_BREAK => \T_BREAK, \T_RETURN => \T_RETURN, \T_CONTINUE => \T_CONTINUE, \T_THROW => \T_THROW], 'strict' => \true, 'shared' => \true, 'with' => [\T_CASE => \T_CASE, \T_SWITCH => \T_SWITCH]]];
/**
* A list of tokens that end the scope.
*
* This array is just a unique collection of the end tokens
* from the _scopeOpeners array. The data is duplicated here to
* save time during parsing of the file.
*
* @var array
*/
public $endScopeTokens = [T_CLOSE_CURLY_BRACKET => T_CLOSE_CURLY_BRACKET, \T_BREAK => \T_BREAK];
/**
* A list of special JS tokens and their types.
*
* @var array
*/
protected $tokenValues = ['class' => 'T_CLASS', 'function' => 'T_FUNCTION', 'prototype' => 'T_PROTOTYPE', 'try' => 'T_TRY', 'catch' => 'T_CATCH', 'return' => 'T_RETURN', 'throw' => 'T_THROW', 'break' => 'T_BREAK', 'switch' => 'T_SWITCH', 'continue' => 'T_CONTINUE', 'if' => 'T_IF', 'else' => 'T_ELSE', 'do' => 'T_DO', 'while' => 'T_WHILE', 'for' => 'T_FOR', 'var' => 'T_VAR', 'case' => 'T_CASE', 'default' => 'T_DEFAULT', 'true' => 'T_TRUE', 'false' => 'T_FALSE', 'null' => 'T_NULL', 'this' => 'T_THIS', 'typeof' => 'T_TYPEOF', '(' => 'T_OPEN_PARENTHESIS', ')' => 'T_CLOSE_PARENTHESIS', '{' => 'T_OPEN_CURLY_BRACKET', '}' => 'T_CLOSE_CURLY_BRACKET', '[' => 'T_OPEN_SQUARE_BRACKET', ']' => 'T_CLOSE_SQUARE_BRACKET', '?' => 'T_INLINE_THEN', '.' => 'T_OBJECT_OPERATOR', '+' => 'T_PLUS', '-' => 'T_MINUS', '*' => 'T_MULTIPLY', '%' => 'T_MODULUS', '/' => 'T_DIVIDE', '^' => 'T_LOGICAL_XOR', ',' => 'T_COMMA', ';' => 'T_SEMICOLON', ':' => 'T_COLON', '<' => 'T_LESS_THAN', '>' => 'T_GREATER_THAN', '<<' => 'T_SL', '>>' => 'T_SR', '>>>' => 'T_ZSR', '<<=' => 'T_SL_EQUAL', '>>=' => 'T_SR_EQUAL', '>>>=' => 'T_ZSR_EQUAL', '<=' => 'T_IS_SMALLER_OR_EQUAL', '>=' => 'T_IS_GREATER_OR_EQUAL', '=>' => 'T_DOUBLE_ARROW', '!' => 'T_BOOLEAN_NOT', '||' => 'T_BOOLEAN_OR', '&&' => 'T_BOOLEAN_AND', '|' => 'T_BITWISE_OR', '&' => 'T_BITWISE_AND', '!=' => 'T_IS_NOT_EQUAL', '!==' => 'T_IS_NOT_IDENTICAL', '=' => 'T_EQUAL', '==' => 'T_IS_EQUAL', '===' => 'T_IS_IDENTICAL', '-=' => 'T_MINUS_EQUAL', '+=' => 'T_PLUS_EQUAL', '*=' => 'T_MUL_EQUAL', '/=' => 'T_DIV_EQUAL', '%=' => 'T_MOD_EQUAL', '++' => 'T_INC', '--' => 'T_DEC', '//' => 'T_COMMENT', '/*' => 'T_COMMENT', '/**' => 'T_DOC_COMMENT', '*/' => 'T_COMMENT'];
/**
* A list string delimiters.
*
* @var array
*/
protected $stringTokens = ['\'' => '\'', '"' => '"'];
/**
* A list tokens that start and end comments.
*
* @var array
*/
protected $commentTokens = ['//' => null, '/*' => '*/', '/**' => '*/'];
/**
* Initialise the tokenizer.
*
* Pre-checks the content to see if it looks minified.
*
* @param string $content The content to tokenize,
* @param \PHP_CodeSniffer\Config $config The config data for the run.
* @param string $eolChar The EOL char used in the content.
*
* @return void
* @throws \PHP_CodeSniffer\Exceptions\TokenizerException If the file appears to be minified.
*/
public function __construct($content, Config $config, $eolChar = '\\n')
{
if ($this->isMinifiedContent($content, $eolChar) === \true) {
throw new TokenizerException('File appears to be minified and cannot be processed');
}
parent::__construct($content, $config, $eolChar);
}
//end __construct()
/**
* Creates an array of tokens when given some JS code.
*
* @param string $string The string to tokenize.
*
* @return array
*/
public function tokenize($string)
{
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t*** START JS TOKENIZING ***" . \PHP_EOL;
}
$maxTokenLength = 0;
foreach ($this->tokenValues as $token => $values) {
if (\strlen($token) > $maxTokenLength) {
$maxTokenLength = \strlen($token);
}
}
$tokens = [];
$inString = '';
$stringChar = null;
$inComment = '';
$buffer = '';
$preStringBuffer = '';
$cleanBuffer = \false;
$commentTokenizer = new Comment();
$tokens[] = ['code' => \T_OPEN_TAG, 'type' => 'T_OPEN_TAG', 'content' => ''];
// Convert newlines to single characters for ease of
// processing. We will change them back later.
$string = \str_replace($this->eolChar, "\n", $string);
$chars = \str_split($string);
$numChars = \count($chars);
for ($i = 0; $i < $numChars; $i++) {
$char = $chars[$i];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($char);
$bufferContent = Util\Common::prepareForOutput($buffer);
if ($inString !== '') {
echo "\t";
}
if ($inComment !== '') {
echo "\t";
}
echo "\tProcess char {$i} => {$content} (buffer: {$bufferContent})" . \PHP_EOL;
}
//end if
if ($inString === '' && $inComment === '' && $buffer !== '') {
// If the buffer only has whitespace and we are about to
// add a character, store the whitespace first.
if (\trim($char) !== '' && \trim($buffer) === '') {
$tokens[] = ['code' => \T_WHITESPACE, 'type' => 'T_WHITESPACE', 'content' => \str_replace("\n", $this->eolChar, $buffer)];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($buffer);
echo "\t=> Added token T_WHITESPACE ({$content})" . \PHP_EOL;
}
$buffer = '';
}
// If the buffer is not whitespace and we are about to
// add a whitespace character, store the content first.
if ($inString === '' && $inComment === '' && \trim($char) === '' && \trim($buffer) !== '') {
$tokens[] = ['code' => \T_STRING, 'type' => 'T_STRING', 'content' => \str_replace("\n", $this->eolChar, $buffer)];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($buffer);
echo "\t=> Added token T_STRING ({$content})" . \PHP_EOL;
}
$buffer = '';
}
}
//end if
// Process strings.
if ($inComment === '' && isset($this->stringTokens[$char]) === \true) {
if ($inString === $char) {
// This could be the end of the string, but make sure it
// is not escaped first.
$escapes = 0;
for ($x = $i - 1; $x >= 0; $x--) {
if ($chars[$x] !== '\\') {
break;
}
$escapes++;
}
if ($escapes === 0 || $escapes % 2 === 0) {
// There is an even number escape chars,
// so this is not escaped, it is the end of the string.
$tokens[] = ['code' => \T_CONSTANT_ENCAPSED_STRING, 'type' => 'T_CONSTANT_ENCAPSED_STRING', 'content' => \str_replace("\n", $this->eolChar, $buffer) . $char];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* found end of string *" . \PHP_EOL;
$content = Util\Common::prepareForOutput($buffer . $char);
echo "\t=> Added token T_CONSTANT_ENCAPSED_STRING ({$content})" . \PHP_EOL;
}
$buffer = '';
$preStringBuffer = '';
$inString = '';
$stringChar = null;
continue;
}
//end if
} else {
if ($inString === '') {
$inString = $char;
$stringChar = $i;
$preStringBuffer = $buffer;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* looking for string closer *" . \PHP_EOL;
}
}
}
//end if
}
//end if
if ($inString !== '' && $char === "\n") {
// Unless this newline character is escaped, the string did not
// end before the end of the line, which means it probably
// wasn't a string at all (maybe a regex).
if ($chars[$i - 1] !== '\\') {
$i = $stringChar;
$buffer = $preStringBuffer;
$preStringBuffer = '';
$inString = '';
$stringChar = null;
$char = $chars[$i];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* found newline before end of string, bailing *" . \PHP_EOL;
}
}
}
$buffer .= $char;
// We don't look for special tokens inside strings,
// so if we are in a string, we can continue here now
// that the current char is in the buffer.
if ($inString !== '') {
continue;
}
// Special case for T_DIVIDE which can actually be
// the start of a regular expression.
if ($buffer === $char && $char === '/' && $chars[$i + 1] !== '*') {
$regex = $this->getRegexToken($i, $string, $chars, $tokens);
if ($regex !== null) {
$tokens[] = ['code' => T_REGULAR_EXPRESSION, 'type' => 'T_REGULAR_EXPRESSION', 'content' => $regex['content']];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($regex['content']);
echo "\t=> Added token T_REGULAR_EXPRESSION ({$content})" . \PHP_EOL;
}
$i = $regex['end'];
$buffer = '';
$cleanBuffer = \false;
continue;
}
//end if
}
//end if
// Check for known tokens, but ignore tokens found that are not at
// the end of a string, like FOR and this.FORmat.
if (isset($this->tokenValues[\strtolower($buffer)]) === \true && (\preg_match('|[a-zA-z0-9_]|', $char) === 0 || isset($chars[$i + 1]) === \false || \preg_match('|[a-zA-z0-9_]|', $chars[$i + 1]) === 0)) {
$matchedToken = \false;
$lookAheadLength = $maxTokenLength - \strlen($buffer);
if ($lookAheadLength > 0) {
// The buffer contains a token type, but we need
// to look ahead at the next chars to see if this is
// actually part of a larger token. For example,
// FOR and FOREACH.
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* buffer possibly contains token, looking ahead {$lookAheadLength} chars *" . \PHP_EOL;
}
$charBuffer = $buffer;
for ($x = 1; $x <= $lookAheadLength; $x++) {
if (isset($chars[$i + $x]) === \false) {
break;
}
$charBuffer .= $chars[$i + $x];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($charBuffer);
echo "\t\t=> Looking ahead {$x} chars => {$content}" . \PHP_EOL;
}
if (isset($this->tokenValues[\strtolower($charBuffer)]) === \true) {
// We've found something larger that matches
// so we can ignore this char. Except for 1 very specific
// case where a comment like /**/ needs to tokenize as
// T_COMMENT and not T_DOC_COMMENT.
$oldType = $this->tokenValues[\strtolower($buffer)];
$newType = $this->tokenValues[\strtolower($charBuffer)];
if ($oldType === 'T_COMMENT' && $newType === 'T_DOC_COMMENT' && $chars[$i + $x + 1] === '/') {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* look ahead ignored T_DOC_COMMENT, continuing *" . \PHP_EOL;
}
} else {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* look ahead found more specific token ({$newType}), ignoring {$i} *" . \PHP_EOL;
}
$matchedToken = \true;
break;
}
}
//end if
}
//end for
}
//end if
if ($matchedToken === \false) {
if (PHP_CODESNIFFER_VERBOSITY > 1 && $lookAheadLength > 0) {
echo "\t\t* look ahead found nothing *" . \PHP_EOL;
}
$value = $this->tokenValues[\strtolower($buffer)];
if ($value === 'T_FUNCTION' && $buffer !== 'function') {
// The function keyword needs to be all lowercase or else
// it is just a function called "Function".
$value = 'T_STRING';
}
$tokens[] = ['code' => \constant($value), 'type' => $value, 'content' => $buffer];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($buffer);
echo "\t=> Added token {$value} ({$content})" . \PHP_EOL;
}
$cleanBuffer = \true;
}
//end if
} else {
if (isset($this->tokenValues[\strtolower($char)]) === \true) {
// No matter what token we end up using, we don't
// need the content in the buffer any more because we have
// found a valid token.
$newContent = \substr(\str_replace("\n", $this->eolChar, $buffer), 0, -1);
if ($newContent !== '') {
$tokens[] = ['code' => \T_STRING, 'type' => 'T_STRING', 'content' => $newContent];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput(\substr($buffer, 0, -1));
echo "\t=> Added token T_STRING ({$content})" . \PHP_EOL;
}
}
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* char is token, looking ahead " . ($maxTokenLength - 1) . ' chars *' . \PHP_EOL;
}
// The char is a token type, but we need to look ahead at the
// next chars to see if this is actually part of a larger token.
// For example, = and ===.
$charBuffer = $char;
$matchedToken = \false;
for ($x = 1; $x <= $maxTokenLength; $x++) {
if (isset($chars[$i + $x]) === \false) {
break;
}
$charBuffer .= $chars[$i + $x];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($charBuffer);
echo "\t\t=> Looking ahead {$x} chars => {$content}" . \PHP_EOL;
}
if (isset($this->tokenValues[\strtolower($charBuffer)]) === \true) {
// We've found something larger that matches
// so we can ignore this char.
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = $this->tokenValues[\strtolower($charBuffer)];
echo "\t\t* look ahead found more specific token ({$type}), ignoring {$i} *" . \PHP_EOL;
}
$matchedToken = \true;
break;
}
}
//end for
if ($matchedToken === \false) {
$value = $this->tokenValues[\strtolower($char)];
$tokens[] = ['code' => \constant($value), 'type' => $value, 'content' => $char];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* look ahead found nothing *" . \PHP_EOL;
$content = Util\Common::prepareForOutput($char);
echo "\t=> Added token {$value} ({$content})" . \PHP_EOL;
}
$cleanBuffer = \true;
} else {
$buffer = $char;
}
//end if
}
}
//end if
// Keep track of content inside comments.
if ($inComment === '' && \array_key_exists($buffer, $this->commentTokens) === \true) {
// This is not really a comment if the content
// looks like \// (i.e., it is escaped).
if (isset($chars[$i - 2]) === \true && $chars[$i - 2] === '\\') {
$lastToken = \array_pop($tokens);
$lastContent = $lastToken['content'];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$value = $this->tokenValues[\strtolower($lastContent)];
$content = Util\Common::prepareForOutput($lastContent);
echo "\t=> Removed token {$value} ({$content})" . \PHP_EOL;
}
$lastChars = \str_split($lastContent);
$lastNumChars = \count($lastChars);
for ($x = 0; $x < $lastNumChars; $x++) {
$lastChar = $lastChars[$x];
$value = $this->tokenValues[\strtolower($lastChar)];
$tokens[] = ['code' => \constant($value), 'type' => $value, 'content' => $lastChar];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($lastChar);
echo "\t=> Added token {$value} ({$content})" . \PHP_EOL;
}
}
} else {
// We have started a comment.
$inComment = $buffer;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* looking for end of comment *" . \PHP_EOL;
}
}
//end if
} else {
if ($inComment !== '') {
if ($this->commentTokens[$inComment] === null) {
// Comment ends at the next newline.
if (\strpos($buffer, "\n") !== \false) {
$inComment = '';
}
} else {
if ($this->commentTokens[$inComment] === $buffer) {
$inComment = '';
}
}
if (PHP_CODESNIFFER_VERBOSITY > 1) {
if ($inComment === '') {
echo "\t\t* found end of comment *" . \PHP_EOL;
}
}
if ($inComment === '' && $cleanBuffer === \false) {
$tokens[] = ['code' => \T_STRING, 'type' => 'T_STRING', 'content' => \str_replace("\n", $this->eolChar, $buffer)];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($buffer);
echo "\t=> Added token T_STRING ({$content})" . \PHP_EOL;
}
$buffer = '';
}
}
}
//end if
if ($cleanBuffer === \true) {
$buffer = '';
$cleanBuffer = \false;
}
}
//end for
if (empty($buffer) === \false) {
if ($inString !== '') {
// The string did not end before the end of the file,
// which means there was probably a syntax error somewhere.
$tokens[] = ['code' => \T_STRING, 'type' => 'T_STRING', 'content' => \str_replace("\n", $this->eolChar, $buffer)];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($buffer);
echo "\t=> Added token T_STRING ({$content})" . \PHP_EOL;
}
} else {
// Buffer contains whitespace from the end of the file.
$tokens[] = ['code' => \T_WHITESPACE, 'type' => 'T_WHITESPACE', 'content' => \str_replace("\n", $this->eolChar, $buffer)];
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$content = Util\Common::prepareForOutput($buffer);
echo "\t=> Added token T_WHITESPACE ({$content})" . \PHP_EOL;
}
}
//end if
}
//end if
$tokens[] = ['code' => \T_CLOSE_TAG, 'type' => 'T_CLOSE_TAG', 'content' => ''];
/*
Now that we have done some basic tokenizing, we need to
modify the tokens to join some together and split some apart
so they match what the PHP tokenizer does.
*/
$finalTokens = [];
$newStackPtr = 0;
$numTokens = \count($tokens);
for ($stackPtr = 0; $stackPtr < $numTokens; $stackPtr++) {
$token = $tokens[$stackPtr];
/*
Look for comments and join the tokens together.
*/
if ($token['code'] === \T_COMMENT || $token['code'] === \T_DOC_COMMENT) {
$newContent = '';
$tokenContent = $token['content'];
$endContent = null;
if (isset($this->commentTokens[$tokenContent]) === \true) {
$endContent = $this->commentTokens[$tokenContent];
}
while ($tokenContent !== $endContent) {
if ($endContent === null && \strpos($tokenContent, $this->eolChar) !== \false) {
// A null end token means the comment ends at the end of
// the line so we look for newlines and split the token.
$tokens[$stackPtr]['content'] = \substr($tokenContent, \strpos($tokenContent, $this->eolChar) + \strlen($this->eolChar));
$tokenContent = \substr($tokenContent, 0, \strpos($tokenContent, $this->eolChar) + \strlen($this->eolChar));
// If the substr failed, skip the token as the content
// will now be blank.
if ($tokens[$stackPtr]['content'] !== \false && $tokens[$stackPtr]['content'] !== '') {
$stackPtr--;
}
break;
}
//end if
$stackPtr++;
$newContent .= $tokenContent;
if (isset($tokens[$stackPtr]) === \false) {
break;
}
$tokenContent = $tokens[$stackPtr]['content'];
}
//end while
if ($token['code'] === \T_DOC_COMMENT) {
$commentTokens = $commentTokenizer->tokenizeString($newContent . $tokenContent, $this->eolChar, $newStackPtr);
foreach ($commentTokens as $commentToken) {
$finalTokens[$newStackPtr] = $commentToken;
$newStackPtr++;
}
continue;
} else {
// Save the new content in the current token so
// the code below can chop it up on newlines.
$token['content'] = $newContent . $tokenContent;
}
}
//end if
/*
If this token has newlines in its content, split each line up
and create a new token for each line. We do this so it's easier
to ascertain where errors occur on a line.
Note that $token[1] is the token's content.
*/
if (\strpos($token['content'], $this->eolChar) !== \false) {
$tokenLines = \explode($this->eolChar, $token['content']);
$numLines = \count($tokenLines);
for ($i = 0; $i < $numLines; $i++) {
$newToken = ['content' => $tokenLines[$i]];
if ($i === $numLines - 1) {
if ($tokenLines[$i] === '') {
break;
}
} else {
$newToken['content'] .= $this->eolChar;
}
$newToken['type'] = $token['type'];
$newToken['code'] = $token['code'];
$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
}
} else {
$finalTokens[$newStackPtr] = $token;
$newStackPtr++;
}
//end if
// Convert numbers, including decimals.
if ($token['code'] === \T_STRING || $token['code'] === \T_OBJECT_OPERATOR) {
$newContent = '';
$oldStackPtr = $stackPtr;
while (\preg_match('|^[0-9\\.]+$|', $tokens[$stackPtr]['content']) !== 0) {
$newContent .= $tokens[$stackPtr]['content'];
$stackPtr++;
}
if ($newContent !== '' && $newContent !== '.') {
$finalTokens[$newStackPtr - 1]['content'] = $newContent;
if (\ctype_digit($newContent) === \true) {
$finalTokens[$newStackPtr - 1]['code'] = \constant('T_LNUMBER');
$finalTokens[$newStackPtr - 1]['type'] = 'T_LNUMBER';
} else {
$finalTokens[$newStackPtr - 1]['code'] = \constant('T_DNUMBER');
$finalTokens[$newStackPtr - 1]['type'] = 'T_DNUMBER';
}
$stackPtr--;
continue;
} else {
$stackPtr = $oldStackPtr;
}
}
//end if
// Convert the token after an object operator into a string, in most cases.
if ($token['code'] === \T_OBJECT_OPERATOR) {
for ($i = $stackPtr + 1; $i < $numTokens; $i++) {
if (isset(Util\Tokens::$emptyTokens[$tokens[$i]['code']]) === \true) {
continue;
}
if ($tokens[$i]['code'] !== T_PROTOTYPE && $tokens[$i]['code'] !== \T_LNUMBER && $tokens[$i]['code'] !== \T_DNUMBER) {
$tokens[$i]['code'] = \T_STRING;
$tokens[$i]['type'] = 'T_STRING';
}
break;
}
}
}
//end for
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t*** END TOKENIZING ***" . \PHP_EOL;
}
return $finalTokens;
}
//end tokenize()
/**
* Tokenizes a regular expression if one is found.
*
* If a regular expression is not found, NULL is returned.
*
* @param int $char The index of the possible regex start character.
* @param string $string The complete content of the string being tokenized.
* @param array $chars An array of characters being tokenized.
* @param array $tokens The current array of tokens found in the string.
*
* @return array<string, string>|null
*/
public function getRegexToken($char, $string, $chars, $tokens)
{
$beforeTokens = [T_EQUAL => \true, \T_IS_NOT_EQUAL => \true, \T_IS_IDENTICAL => \true, \T_IS_NOT_IDENTICAL => \true, T_OPEN_PARENTHESIS => \true, T_OPEN_SQUARE_BRACKET => \true, \T_RETURN => \true, \T_BOOLEAN_OR => \true, \T_BOOLEAN_AND => \true, T_BOOLEAN_NOT => \true, T_BITWISE_OR => \true, T_BITWISE_AND => \true, T_COMMA => \true, T_COLON => \true, T_TYPEOF => \true, T_INLINE_THEN => \true, T_INLINE_ELSE => \true];
$afterTokens = [',' => \true, ')' => \true, ']' => \true, ';' => \true, ' ' => \true, '.' => \true, ':' => \true, $this->eolChar => \true];
// Find the last non-whitespace token that was added
// to the tokens array.
$numTokens = \count($tokens);
for ($prev = $numTokens - 1; $prev >= 0; $prev--) {
if (isset(Util\Tokens::$emptyTokens[$tokens[$prev]['code']]) === \false) {
break;
}
}
if (isset($beforeTokens[$tokens[$prev]['code']]) === \false) {
return null;
}
// This is probably a regular expression, so look for the end of it.
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t* token possibly starts a regular expression *" . \PHP_EOL;
}
$numChars = \count($chars);
for ($next = $char + 1; $next < $numChars; $next++) {
if ($chars[$next] === '/') {
// Just make sure this is not escaped first.
if ($chars[$next - 1] !== '\\') {
// In the simple form: /.../ so we found the end.
break;
} else {
if ($chars[$next - 2] === '\\') {
// In the form: /...\\/ so we found the end.
break;
}
}
} else {
$possibleEolChar = \substr($string, $next, \strlen($this->eolChar));
if ($possibleEolChar === $this->eolChar) {
// This is the last token on the line and regular
// expressions need to be defined on a single line,
// so this is not a regular expression.
break;
}
}
}
if ($chars[$next] !== '/') {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t* could not find end of regular expression *" . \PHP_EOL;
}
return null;
}
while (\preg_match('|[a-zA-Z]|', $chars[$next + 1]) !== 0) {
// The token directly after the end of the regex can
// be modifiers like global and case insensitive
// (.e.g, /pattern/gi).
$next++;
}
$regexEnd = $next;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t* found end of regular expression at token {$regexEnd} *" . \PHP_EOL;
}
for ($next += 1; $next < $numChars; $next++) {
if ($chars[$next] !== ' ') {
break;
} else {
$possibleEolChar = \substr($string, $next, \strlen($this->eolChar));
if ($possibleEolChar === $this->eolChar) {
// This is the last token on the line.
break;
}
}
}
if (isset($afterTokens[$chars[$next]]) === \false) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t* tokens after regular expression do not look correct *" . \PHP_EOL;
}
return null;
}
// This is a regular expression, so join all the tokens together.
$content = '';
for ($x = $char; $x <= $regexEnd; $x++) {
$content .= $chars[$x];
}
$token = ['start' => $char, 'end' => $regexEnd, 'content' => $content];
return $token;
}
//end getRegexToken()
/**
* Performs additional processing after main tokenizing.
*
* This additional processing looks for properties, closures, labels and objects.
*
* @return void
*/
public function processAdditional()
{
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t*** START ADDITIONAL JS PROCESSING ***" . \PHP_EOL;
}
$numTokens = \count($this->tokens);
$classStack = [];
for ($i = 0; $i < $numTokens; $i++) {
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = $this->tokens[$i]['type'];
$content = Util\Common::prepareForOutput($this->tokens[$i]['content']);
echo \str_repeat("\t", \count($classStack));
echo "\tProcess token {$i}: {$type} => {$content}" . \PHP_EOL;
}
// Looking for functions that are actually closures.
if ($this->tokens[$i]['code'] === \T_FUNCTION && isset($this->tokens[$i]['scope_opener']) === \true) {
for ($x = $i + 1; $x < $numTokens; $x++) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$x]['code']]) === \false) {
break;
}
}
if ($this->tokens[$x]['code'] === T_OPEN_PARENTHESIS) {
$this->tokens[$i]['code'] = T_CLOSURE;
$this->tokens[$i]['type'] = 'T_CLOSURE';
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$line = $this->tokens[$i]['line'];
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$i} on line {$line} changed from T_FUNCTION to T_CLOSURE *" . \PHP_EOL;
}
for ($x = $this->tokens[$i]['scope_opener'] + 1; $x < $this->tokens[$i]['scope_closer']; $x++) {
if (isset($this->tokens[$x]['conditions'][$i]) === \false) {
continue;
}
$this->tokens[$x]['conditions'][$i] = T_CLOSURE;
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = $this->tokens[$x]['type'];
echo \str_repeat("\t", \count($classStack));
echo "\t\t* cleaned {$x} ({$type}) *" . \PHP_EOL;
}
}
}
//end if
continue;
} else {
if ($this->tokens[$i]['code'] === T_OPEN_CURLY_BRACKET && isset($this->tokens[$i]['scope_condition']) === \false && isset($this->tokens[$i]['bracket_closer']) === \true) {
$condition = $this->tokens[$i]['conditions'];
$condition = \end($condition);
if ($condition === \T_CLASS) {
// Possibly an ES6 method. To be classified as one, the previous
// non-empty tokens need to be a set of parenthesis, and then a string
// (the method name).
for ($parenCloser = $i - 1; $parenCloser > 0; $parenCloser--) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$parenCloser]['code']]) === \false) {
break;
}
}
if ($this->tokens[$parenCloser]['code'] === T_CLOSE_PARENTHESIS) {
$parenOpener = $this->tokens[$parenCloser]['parenthesis_opener'];
for ($name = $parenOpener - 1; $name > 0; $name--) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$name]['code']]) === \false) {
break;
}
}
if ($this->tokens[$name]['code'] === \T_STRING) {
// We found a method name.
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$line = $this->tokens[$name]['line'];
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$name} on line {$line} changed from T_STRING to T_FUNCTION *" . \PHP_EOL;
}
$closer = $this->tokens[$i]['bracket_closer'];
$this->tokens[$name]['code'] = \T_FUNCTION;
$this->tokens[$name]['type'] = 'T_FUNCTION';
foreach ([$name, $i, $closer] as $token) {
$this->tokens[$token]['scope_condition'] = $name;
$this->tokens[$token]['scope_opener'] = $i;
$this->tokens[$token]['scope_closer'] = $closer;
$this->tokens[$token]['parenthesis_opener'] = $parenOpener;
$this->tokens[$token]['parenthesis_closer'] = $parenCloser;
$this->tokens[$token]['parenthesis_owner'] = $name;
}
$this->tokens[$parenOpener]['parenthesis_owner'] = $name;
$this->tokens[$parenCloser]['parenthesis_owner'] = $name;
for ($x = $i + 1; $x < $closer; $x++) {
$this->tokens[$x]['conditions'][$name] = \T_FUNCTION;
\ksort($this->tokens[$x]['conditions'], \SORT_NUMERIC);
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = $this->tokens[$x]['type'];
echo \str_repeat("\t", \count($classStack));
echo "\t\t* added T_FUNCTION condition to {$x} ({$type}) *" . \PHP_EOL;
}
}
continue;
}
//end if
}
//end if
}
//end if
$classStack[] = $i;
$closer = $this->tokens[$i]['bracket_closer'];
$this->tokens[$i]['code'] = T_OBJECT;
$this->tokens[$i]['type'] = 'T_OBJECT';
$this->tokens[$closer]['code'] = T_CLOSE_OBJECT;
$this->tokens[$closer]['type'] = 'T_CLOSE_OBJECT';
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$i} converted from T_OPEN_CURLY_BRACKET to T_OBJECT *" . \PHP_EOL;
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$closer} converted from T_CLOSE_CURLY_BRACKET to T_CLOSE_OBJECT *" . \PHP_EOL;
}
for ($x = $i + 1; $x < $closer; $x++) {
$this->tokens[$x]['conditions'][$i] = T_OBJECT;
\ksort($this->tokens[$x]['conditions'], \SORT_NUMERIC);
if (PHP_CODESNIFFER_VERBOSITY > 1) {
$type = $this->tokens[$x]['type'];
echo \str_repeat("\t", \count($classStack));
echo "\t\t* added T_OBJECT condition to {$x} ({$type}) *" . \PHP_EOL;
}
}
} else {
if ($this->tokens[$i]['code'] === T_CLOSE_OBJECT) {
$opener = \array_pop($classStack);
} else {
if ($this->tokens[$i]['code'] === T_COLON) {
// If it is a scope opener, it belongs to a
// DEFAULT or CASE statement.
if (isset($this->tokens[$i]['scope_condition']) === \true) {
continue;
}
// Make sure this is not part of an inline IF statement.
for ($x = $i - 1; $x >= 0; $x--) {
if ($this->tokens[$x]['code'] === T_INLINE_THEN) {
$this->tokens[$i]['code'] = T_INLINE_ELSE;
$this->tokens[$i]['type'] = 'T_INLINE_ELSE';
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$i} converted from T_COLON to T_INLINE_THEN *" . \PHP_EOL;
}
continue 2;
} else {
if ($this->tokens[$x]['line'] < $this->tokens[$i]['line']) {
break;
}
}
}
// The string to the left of the colon is either a property or label.
for ($label = $i - 1; $label >= 0; $label--) {
if (isset(Util\Tokens::$emptyTokens[$this->tokens[$label]['code']]) === \false) {
break;
}
}
if ($this->tokens[$label]['code'] !== \T_STRING && $this->tokens[$label]['code'] !== \T_CONSTANT_ENCAPSED_STRING) {
continue;
}
if (empty($classStack) === \false) {
$this->tokens[$label]['code'] = T_PROPERTY;
$this->tokens[$label]['type'] = 'T_PROPERTY';
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$label} converted from T_STRING to T_PROPERTY *" . \PHP_EOL;
}
} else {
$this->tokens[$label]['code'] = T_LABEL;
$this->tokens[$label]['type'] = 'T_LABEL';
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo \str_repeat("\t", \count($classStack));
echo "\t* token {$label} converted from T_STRING to T_LABEL *" . \PHP_EOL;
}
}
//end if
}
}
}
}
//end if
}
//end for
if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t*** END ADDITIONAL JS PROCESSING ***" . \PHP_EOL;
}
}
//end processAdditional()
}
//end class