protected function getEncodableWordTokens($string)
{
$tokens = array();
$encodedToken = '';
// Split at all whitespace boundaries
foreach (preg_split('~(?=[\\t ])~', $string) as $token) {
if ($this->tokenNeedsEncoding($token)) {
$encodedToken .= $token;
} else {
if (strlen($encodedToken) > 0) {
$tokens[] = $encodedToken;
$encodedToken = '';
}
$tokens[] = $token;
}
}
if (strlen($encodedToken)) {
$tokens[] = $encodedToken;
}
return $tokens;
}