protected getEncodableWordTokens ( string $string ) : string[] | ||
$string | string | |
return | string[] |
protected function getEncodableWordTokens($string)
{
$tokens = array();
$encodedToken = '';
// Split at all whitespace boundaries
foreach (preg_split('~(?=[\\t ])~', $string) as $token) {
if ($this->tokenNeedsEncoding($token)) {
$encodedToken .= $token;
} else {
if (strlen($encodedToken) > 0) {
$tokens[] = $encodedToken;
$encodedToken = '';
}
$tokens[] = $token;
}
}
if (strlen($encodedToken)) {
$tokens[] = $encodedToken;
}
return $tokens;
}