public function testLex($code, $options, $tokens)
{
$lexer = $this->getLexer($options);
$lexer->startLexing($code);
while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
$token = array_shift($tokens);
$this->assertSame($token[0], $id);
$this->assertSame($token[1], $value);
$this->assertEquals($token[2], $startAttributes);
$this->assertEquals($token[3], $endAttributes);
}
}