PhpPeg\Rule::tokenize PHP Метод

    function tokenize($str, &$tokens, $o = 0)
    {
        $pending = new Pending();
        while ($o < strlen($str)) {
            $sub = substr($str, $o);
            /* Absorb white-space */
            if (preg_match('/^\\s+/', $sub, $match)) {
                $o += strlen($match[0]);
            } elseif (preg_match('/^(\\w*):/', $sub, $match)) {
                $pending->set('tag', isset($match[1]) ? $match[1] : '');
                $o += strlen($match[0]);
            } elseif (preg_match('/^[\\w-]+/', $sub, $match)) {
                $tokens[] = $t = new TokenRecurse($match[0]);
                $pending->apply_if_present($t);
                $o += strlen($match[0]);
            } elseif (preg_match('/^"[^"]*"/', $sub, $match)) {
                $tokens[] = $t = new TokenLiteral($match[0]);
                $pending->apply_if_present($t);
                $o += strlen($match[0]);
            } elseif (preg_match("/^'[^']*'/", $sub, $match)) {
                $tokens[] = $t = new TokenLiteral($match[0]);
                $pending->apply_if_present($t);
                $o += strlen($match[0]);
            } elseif (preg_match(self::$rx_rx, $sub, $match)) {
                $tokens[] = $t = new TokenRegex($match[0]);
                $pending->apply_if_present($t);
                $o += strlen($match[0]);
            } elseif (preg_match('/^\\$(\\w+)/', $sub, $match)) {
                $tokens[] = $t = new TokenExpressionedRecurse($match[1]);
                $pending->apply_if_present($t);
                $o += strlen($match[0]);
            } elseif (preg_match('/^\\@(\\w+)/', $sub, $match)) {
                $l = count($tokens) - 1;
                $o += strlen($match[0]);
                user_error("TODO: Flags not currently supported", E_USER_WARNING);
            } else {
                $c = substr($sub, 0, 1);
                $l = count($tokens) - 1;
                $o += 1;
                switch ($c) {
                    case '?':
                        $tokens[$l]->optional = TRUE;
                        break;
                    case '*':
                        $tokens[$l]->zero_or_more = TRUE;
                        break;
                    case '+':
                        $tokens[$l]->one_or_more = TRUE;
                        break;
                    case '&':
                        $pending->set('positive_lookahead');
                        break;
                    case '!':
                        $pending->set('negative_lookahead');
                        break;
                    case '.':
                        $pending->set('silent');
                        break;
                    case '[':
                    case ']':
                        $tokens[] = new TokenWhitespace(FALSE);
                        break;
                    case '<':
                    case '>':
                        $tokens[] = new TokenWhitespace(TRUE);
                        break;
                    case '(':
                        $subtokens = [];
                        $o = $this->tokenize($str, $subtokens, $o);
                        $tokens[] = $t = new TokenSequence($subtokens);
                        $pending->apply_if_present($t);
                        break;
                    case ')':
                        return $o;
                    case '|':
                        $option1 = $tokens;
                        $option2 = [];
                        $o = $this->tokenize($str, $option2, $o);
                        $option1 = count($option1) == 1 ? $option1[0] : new TokenSequence($option1);
                        $option2 = count($option2) == 1 ? $option2[0] : new TokenSequence($option2);
                        $pending->apply_if_present($option2);
                        $tokens = [new TokenOption($option1, $option2)];
                        return $o;
                    default:
                        user_error("Can't parser {$c} - attempting to skip", E_USER_WARNING);
                }
            }
        }
        return $o;
    }