TeamTNT\TNTSearch\Indexer\TNTIndexer::breakIntoTokens PHP Method

    public function breakIntoTokens($text)
    {
        if ($this->decodeHTMLEntities) {
            $text = html_entity_decode($text);
        }
        return $this->tokenizer->tokenize($text);
    }

 public function testSetTokenizer()
 {
     $someTokenizer = new SomeTokenizer();
     $indexer = new TNTIndexer();
     $indexer->setTokenizer($someTokenizer);
     $this->assertInstanceOf(TokenizerInterface::class, $indexer->tokenizer);
     $res = $indexer->breakIntoTokens('Canon 70-200');
     $this->assertContains("canon", $res);
     $this->assertContains("70-200", $res);
 }