mageekguy\atoum\php\tokenizer::tokenize PHP Method

    public function tokenize($string)
    {
        $this->currentIterator = $this->iterator;
        foreach ($this->tokens = new \arrayIterator(token_get_all($string)) as $key => $token) {
            switch ($token[0]) {
                case T_CONST:
                    $token = $this->appendConstant();
                    break;
                case T_USE:
                    $token = $this->appendImportation();
                    break;
                case T_NAMESPACE:
                    $token = $this->appendNamespace();
                    break;
                case T_FUNCTION:
                    $token = $this->appendFunction();
                    break;
            }
            $this->currentIterator->append(new token($token[0], isset($token[1]) === false ? null : $token[1], isset($token[2]) === false ? null : $token[2]));
        }
        return $this;
    }