Tokenizer::_tokenize

private function _tokenize($sql)
{
	$this->tokens = array();
	$tokenCount = 0;
	$chars = '(),.:=;/';
	$rawTokens = preg_split("/(
		[ \t\n\r]+                   # WHITESPACE
		|\\+                          # BACKSLASHES
		|"                             # DOUBLE QUOTE
		|'                              # SINGLE QUOTE
		|`[^`]+`                        # BACK QUOTE
		|\[[^\]]+\]                  # SQUARE QUOTE
		|\/\*.*?\*\/                # COMMENTARY
		|--.*?\n                       # COMMENTARY
		|\#[^']*?\n                   # COMMENTARY
		|[".preg_quote($chars, "/")."]  # CHARACTER
	)/xs", $sql, -1, PREG_SPLIT_DELIM_CAPTURE);
	$isInSingleQuote = false;
	$isInDoubleQuote = false;
	foreach ($rawTokens as $i => $rawToken)
	{
		if ($rawToken === "")
			continue;

		/** @var Token $prevToken */
		$prevToken = $this->tokens[$tokenCount-1] ?? null;

		if ($isInSingleQuote)
		{
			$prevToken->appendText($rawToken);
			if (
				$rawToken === "'"
				&& preg_match("/(\\)*'$/", $prevToken->text, $match)
				&& (
					mb_strlen($match[0]) === 0
					|| (mb_strlen($match[0]) % 2) === 1
				)
			)
			{
				$isInSingleQuote = false;
			}
		}
		elseif ($isInDoubleQuote)
		{
			$prevToken->appendText($rawToken);
			if (
				$rawToken === """
				&& preg_match("/(\\)*"$/", $prevToken->text, $match)
				&& (mb_strlen($match[0]) % 2) === 1
			)
			{
				$isInDoubleQuote = false;
			}
		}
		elseif ($rawToken[0] === "`")
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_BACK_QUOTE, $rawToken);
		}
		elseif ($rawToken[0] === "[")
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_SQUARE_QUOTE, $rawToken);
		}
		elseif (
			($rawToken[0] === "/" && $rawToken[1] === '*')
			|| ($rawToken[0] === "-" && $rawToken[1] === '-')
			|| ($rawToken[0] === "#")
		)
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_COMMENT, $rawToken);
		}
		elseif (mb_strlen($rawToken) == 1 && mb_strpos($chars, $rawToken) !== false)
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_CHAR, $rawToken);
		}
		elseif ($rawToken === """)
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_DOUBLE_QUOTE, $rawToken);
			$isInDoubleQuote = true;
		}
		elseif ($rawToken === "'")
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_SINGLE_QUOTE, $rawToken);
			$isInSingleQuote = true;
		}
		elseif (preg_match("/^[ \t\n\r]+$/", $rawToken))
		{
			$this->tokens[$tokenCount++] = new Token(Token::T_WHITESPACE, $rawToken);
		}
		else
		{
			if ($tokenCount > 0 && $prevToken->type === Token::T_STRING)
			{
				$prevToken->appendText($rawToken);
			}
			else
			{
				$this->tokens[$tokenCount++] = new Token(Token::T_STRING, $rawToken);
			}
		}
	}
}