function _getToken()
{
if (!empty($this->_tokenStack)) {
return array_pop($this->_tokenStack);
}
if ($this->_pos >= $this->_len) {
return NULL;
}
if ($this->_state != -1 && preg_match($this->_endpattern, $this->_str, $m, PREG_OFFSET_CAPTURE, $this->_pos)) {
$endpos = $m[0][1];
$endmatch = $m[0][0];
} else {
$endpos = -1;
}
preg_match($this->_regs[$this->_state], $this->_str, $m, PREG_OFFSET_CAPTURE, $this->_pos);
$n = 1;
foreach ($this->_counts[$this->_state] as $i => $count) {
if (!isset($m[$n])) {
break;
}
if ($m[$n][1] > -1 && ($endpos == -1 || $m[$n][1] < $endpos)) {
if ($this->_states[$this->_state][$i] != -1) {
$this->_tokenStack[] = array($this->_delim[$this->_state][$i], $m[$n][0]);
} else {
$inner = $this->_inner[$this->_state][$i];
if (isset($this->_parts[$this->_state][$i])) {
$parts = array();
$partpos = $m[$n][1];
for ($j = 1; $j <= $count; $j++) {
if ($m[$j + $n][1] < 0) {
continue;
}
if (isset($this->_parts[$this->_state][$i][$j])) {
if ($m[$j + $n][1] > $partpos) {
array_unshift($parts, array($inner, substr($this->_str, $partpos, $m[$j + $n][1] - $partpos)));
}
array_unshift($parts, array($this->_parts[$this->_state][$i][$j], $m[$j + $n][0]));
}
$partpos = $m[$j + $n][1] + strlen($m[$j + $n][0]);
}
if ($partpos < $m[$n][1] + strlen($m[$n][0])) {
array_unshift($parts, array($inner, substr($this->_str, $partpos, $m[$n][1] - $partpos + strlen($m[$n][0]))));
}
$this->_tokenStack = array_merge($this->_tokenStack, $parts);
} else {
foreach ($this->_keywords[$this->_state][$i] as $g => $re) {
if (isset($this->_disabled[$g])) {
continue;
}
if (preg_match($re, $m[$n][0])) {
$inner = $this->_kwmap[$g];
break;
}
}
$this->_tokenStack[] = array($inner, $m[$n][0]);
}
}
if ($m[$n][1] > $this->_pos) {
$this->_tokenStack[] = array($this->_lastinner, substr($this->_str, $this->_pos, $m[$n][1] - $this->_pos));
}
$this->_pos = $m[$n][1] + strlen($m[$n][0]);
if ($this->_states[$this->_state][$i] != -1) {
$this->_stack[] = array($this->_state, $this->_lastdelim, $this->_lastinner, $this->_endpattern);
$this->_lastinner = $this->_inner[$this->_state][$i];
$this->_lastdelim = $this->_delim[$this->_state][$i];
$l = $this->_state;
$this->_state = $this->_states[$this->_state][$i];
$this->_endpattern = $this->_end[$this->_state];
if ($this->_subst[$l][$i]) {
for ($k = 0; $k <= $this->_counts[$l][$i]; $k++) {
if (!isset($m[$i + $k])) {
break;
}
$quoted = preg_quote($m[$n + $k][0], '/');
$this->_endpattern = str_replace('%' . $k . '%', $quoted, $this->_endpattern);
$this->_endpattern = str_replace('%b' . $k . '%', $this->_matchingBrackets($quoted), $this->_endpattern);
}
}
}
return array_pop($this->_tokenStack);
}
$n += $count + 1;
}
if ($endpos > -1) {
$this->_tokenStack[] = array($this->_lastdelim, $endmatch);
if ($endpos > $this->_pos) {
$this->_tokenStack[] = array($this->_lastinner, substr($this->_str, $this->_pos, $endpos - $this->_pos));
}
list($this->_state, $this->_lastdelim, $this->_lastinner, $this->_endpattern) = array_pop($this->_stack);
$this->_pos = $endpos + strlen($endmatch);
return array_pop($this->_tokenStack);
}
$p = $this->_pos;
$this->_pos = HL_INFINITY;
return array($this->_lastinner, substr($this->_str, $p));
}