public function lex(string $value) : array
{
$offset = 0;
$valueLength = strlen($value);
$tokens = [];
while ($offset < $valueLength) {
$valueFragment = substr($value, $offset);
$fragmentTokens = $this->lexFragment($this->referenceLexer, $valueFragment);
foreach ($fragmentTokens as $fragmentToken) {
$tokens[] = $fragmentToken;
$offset += strlen($fragmentToken->getValue());
}
}
return $tokens;
}