public function tokenize(string $value) : string
{
$tokens = $this->tokenizer->tokenize($value);
$tree = $this->buildTree($value, $tokens);
$tokenizedValue = '';
foreach ($tree as $node) {
$tokenizedValue .= $this->tokenizer->isOpeningToken($node) ? sprintf('<aliceTokenizedFunction(%s)>', $node) : $node;
}
return $tokenizedValue;
}
/** * {@inheritdoc} * * @throws MalformedFunctionException */ public function lex(string $value) : array { if (false === $this->functionTokenizer->isTokenized($value)) { $value = $this->functionTokenizer->tokenize($value); } return $this->decoratedLexer->lex($value); }