public function breakIntoTokens($text)
{
if ($this->decodeHTMLEntities) {
$text = html_entity_decode($text);
}
return $this->tokenizer->tokenize($text);
}
public function testSetTokenizer() { $someTokenizer = new SomeTokenizer(); $indexer = new TNTIndexer(); $indexer->setTokenizer($someTokenizer); $this->assertInstanceOf(TokenizerInterface::class, $indexer->tokenizer); $res = $indexer->breakIntoTokens('Canon 70-200'); $this->assertContains("canon", $res); $this->assertContains("70-200", $res); }