Arachnid\Crawler::traverseSingle PHP Метод

    protected function traverseSingle($url, $depth)
    {
        try {
            $client = $this->getScrapClient();
            $crawler = $client->request('GET', $url);
            $statusCode = $client->getResponse()->getStatus();
            $hash = $this->getPathFromUrl($url);
            $this->links[$hash]['status_code'] = $statusCode;
            if ($statusCode === 200) {
                $content_type = $client->getResponse()->getHeader('Content-Type');
                if (strpos($content_type, 'text/html') !== false) {
                    //traverse children in case the response in HTML document only
                    $this->extractTitleInfo($crawler, $hash);
                    $childLinks = array();
                    if (isset($this->links[$hash]['external_link']) === true && $this->links[$hash]['external_link'] === false) {
                        $childLinks = $this->extractLinksInfo($crawler, $hash);
                    }
                    $this->links[$hash]['visited'] = true;
                    $this->traverseChildren($childLinks, $depth - 1);
                }
            }
        } catch (CurlException $e) {
            $this->links[$url]['status_code'] = '404';
            $this->links[$url]['error_code'] = $e->getCode();
            $this->links[$url]['error_message'] = $e->getMessage();
        } catch (\Exception $e) {
            $this->links[$url]['status_code'] = '404';
            $this->links[$url]['error_code'] = $e->getCode();
            $this->links[$url]['error_message'] = $e->getMessage();
        }
    }