public function lex($string)
{
$tokens = array();
$offset = 0;
while (isset($string[$offset])) {
foreach (self::$tokenMap as $regex => $token) {
if (preg_match($regex, $string, $matches, null, $offset)) {
$tokens[] = TokenObject::create($token, trim($matches[0]));
$offset += strlen($matches[0]);
continue 2;
}
}
throw new Exception(sprintf('Unexpected character: >%s< offset >%d<', $string[$offset], $offset));
}
$tokens[] = TokenObject::create(Token::EOF, 'eof');
return $tokens;
}