protected function getTokens($pattern)
{
$char = null;
$tokens = array();
$token = null;
$text = false;
$pattern = preg_replace("/''/", '``````', $pattern);
for ($i = 0; $i < strlen($pattern); $i++) {
if ($char == null || $pattern[$i] == $char || $text) {
$token .= $pattern[$i];
} else {
$tokens[] = str_replace("", "'", $token);
$token = $pattern[$i];
}
if ($pattern[$i] == "'" && $text == false) {
$text = true;
} else {
if ($text && $pattern[$i] == "'" && $char == "'") {
$text = true;
} else {
if ($text && $char != "'" && $pattern[$i] == "'") {
$text = false;
}
}
}
$char = $pattern[$i];
}
$tokens[] = $token;
return $tokens;
}