function tokenize($str, &$tokens, $o = 0)
{
$pending = new Pending();
while ($o < strlen($str)) {
$sub = substr($str, $o);
/* Absorb white-space */
if (preg_match('/^\\s+/', $sub, $match)) {
$o += strlen($match[0]);
} elseif (preg_match('/^(\\w*):/', $sub, $match)) {
$pending->set('tag', isset($match[1]) ? $match[1] : '');
$o += strlen($match[0]);
} elseif (preg_match('/^[\\w-]+/', $sub, $match)) {
$tokens[] = $t = new TokenRecurse($match[0]);
$pending->apply_if_present($t);
$o += strlen($match[0]);
} elseif (preg_match('/^"[^"]*"/', $sub, $match)) {
$tokens[] = $t = new TokenLiteral($match[0]);
$pending->apply_if_present($t);
$o += strlen($match[0]);
} elseif (preg_match("/^'[^']*'/", $sub, $match)) {
$tokens[] = $t = new TokenLiteral($match[0]);
$pending->apply_if_present($t);
$o += strlen($match[0]);
} elseif (preg_match(self::$rx_rx, $sub, $match)) {
$tokens[] = $t = new TokenRegex($match[0]);
$pending->apply_if_present($t);
$o += strlen($match[0]);
} elseif (preg_match('/^\\$(\\w+)/', $sub, $match)) {
$tokens[] = $t = new TokenExpressionedRecurse($match[1]);
$pending->apply_if_present($t);
$o += strlen($match[0]);
} elseif (preg_match('/^\\@(\\w+)/', $sub, $match)) {
$l = count($tokens) - 1;
$o += strlen($match[0]);
user_error("TODO: Flags not currently supported", E_USER_WARNING);
} else {
$c = substr($sub, 0, 1);
$l = count($tokens) - 1;
$o += 1;
switch ($c) {
case '?':
$tokens[$l]->optional = TRUE;
break;
case '*':
$tokens[$l]->zero_or_more = TRUE;
break;
case '+':
$tokens[$l]->one_or_more = TRUE;
break;
case '&':
$pending->set('positive_lookahead');
break;
case '!':
$pending->set('negative_lookahead');
break;
case '.':
$pending->set('silent');
break;
case '[':
case ']':
$tokens[] = new TokenWhitespace(FALSE);
break;
case '<':
case '>':
$tokens[] = new TokenWhitespace(TRUE);
break;
case '(':
$subtokens = [];
$o = $this->tokenize($str, $subtokens, $o);
$tokens[] = $t = new TokenSequence($subtokens);
$pending->apply_if_present($t);
break;
case ')':
return $o;
case '|':
$option1 = $tokens;
$option2 = [];
$o = $this->tokenize($str, $option2, $o);
$option1 = count($option1) == 1 ? $option1[0] : new TokenSequence($option1);
$option2 = count($option2) == 1 ? $option2[0] : new TokenSequence($option2);
$pending->apply_if_present($option2);
$tokens = [new TokenOption($option1, $option2)];
return $o;
default:
user_error("Can't parser {$c} - attempting to skip", E_USER_WARNING);
}
}
}
return $o;
}