protected function toTokens($string = null)
{
if (is_null($string)) {
$string = $this->getFieldBody();
}
$tokens = array();
// Generate atoms; split at all invisible boundaries followed by WSP
foreach (preg_split('~(?=[ \\t])~', $string) as $token) {
$newTokens = $this->generateTokenLines($token);
foreach ($newTokens as $newToken) {
$tokens[] = $newToken;
}
}
return $tokens;
}