Spatie\Crawler\Crawler::crawlUrl PHP Method

    protected function crawlUrl(Url $url)
    {
        if (!$this->crawlProfile->shouldCrawl($url)) {
            return;
        }
        if ($this->hasAlreadyCrawled($url)) {
            return;
        }
        $this->crawlObserver->willCrawl($url);
        try {
            $response = $this->client->request('GET', (string) $url);
        } catch (RequestException $exception) {
            $response = $exception->getResponse();
        }
        $this->crawlObserver->hasBeenCrawled($url, $response);
        $this->crawledUrls->push($url);
        if (!$response) {
            return;
        }
        if ($url->host === $this->baseUrl->host) {
            $this->crawlAllLinks($response->getBody()->getContents());
        }
    }