phplinter\Tokenizer::tokenize PHP Méthode

    public static function tokenize($file, $skip_whitespace = true)
    {
        $eol = PHP_EOL;
        $all = token_get_all(file_get_contents($file));
        $lnum = 1;
        $out = array();
        foreach ($all as $token) {
            // Make all tokens arrays
            if (!is_array($token)) {
                $token = array(self::tokenChar($token), $token, $lnum);
            }
            // Find newlines and mark
            if (mb_strpos($token[1], $eol) !== false) {
                $string = str_split($token[1]);
                $substr = '';
                foreach ($string as $_) {
                    if ($_ === $eol) {
                        if (!empty($substr)) {
                            $tkn = array($token[0], $substr, $token[2]);
                            $out[] = $tkn;
                        }
                        $tkn = array(T_NEWLINE, $eol, $lnum);
                        $out[] = $tkn;
                        $substr = '';
                        $lnum++;
                    } else {
                        $substr .= $_;
                    }
                }
                if (!empty($substr)) {
                    $tkn = array($token[0], $substr, $token[2]);
                    $out[] = $tkn;
                }
                continue;
            }
            // Make distinction
            if ($token[0] == T_STRING) {
                $token[0] = self::tokenString($token[1]);
            }
            // Save token
            $out[] = $token;
        }
        return $skip_whitespace ? self::strip_whitespace($out) : $out;
    }