Phpml\Tokenization\Tokenizer::tokenize PHP Method

    public function tokenize(string $text) : array;

 /**
  * @param string $sample
  */
 private function transformSample(string &$sample)
 {
     $counts = [];
     $tokens = $this->tokenizer->tokenize($sample);
     foreach ($tokens as $token) {
         $index = $this->getTokenIndex($token);
         if (false !== $index) {
             $this->updateFrequency($token);
             if (!isset($counts[$index])) {
                 $counts[$index] = 0;
             }
             ++$counts[$index];
         }
     }
     foreach ($this->vocabulary as $index) {
         if (!isset($counts[$index])) {
             $counts[$index] = 0;
         }
     }
     ksort($counts);
     $sample = $counts;
 }