eZ\Publish\Core\Persistence\TransformationProcessor\DefinitionBased\Parser::tokenize PHP Метод

    protected function tokenize($string)
    {
        $string = preg_replace('(\\r\\n|\\r)', "\n", $string);
        $tokens = array();
        $line = 1;
        while (strlen($string)) {
            foreach ($this->tokenSpecifications as $token => $regexp) {
                if (!preg_match($regexp, $string, $matches)) {
                    continue;
                }
                // Remove matched string
                $string = substr($string, strlen($matches[0]));
                $line += substr_count($matches[0], "\n");
                // Append token to list
                $tokens[] = array('type' => $token, 'data' => $this->filterValues($matches));
                // Continue with outer loop
                continue 2;
            }
            throw new RuntimeException("Parse error in line {$line}: " . substr($string, 0, 100));
        }
        return $tokens;
    }