protected function lex($string)
{
$delimiter = ' ';
$fragment = strtok($string, $delimiter);
$regex = $this->compileRegex();
$tokens = array();
while (false !== $fragment) {
if (preg_match($regex, $fragment, $matches)) {
foreach ($matches as $offset => $val) {
if (!empty($val) && $offset > 0) {
$token = array_values($this->tokenMap)[$offset - 1];
$tokens[] = array('token' => $token, 'value' => strtolower($matches[0]));
}
}
}
$fragment = strtok($delimiter);
}
return $tokens;
}