Spatie\Crawler\Crawler::getCrawlRequests PHP Метод

    protected function getCrawlRequests() : Generator
    {
        $i = 0;
        while ($crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($i)) {
            if (!$this->crawlProfile->shouldCrawl($crawlUrl->url)) {
                $i++;
                continue;
            }
            if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) {
                $i++;
                continue;
            }
            $this->crawlObserver->willCrawl($crawlUrl->url);
            $this->crawlQueue->markAsProcessed($crawlUrl);
            (yield new Request('GET', (string) $crawlUrl->url));
            $i++;
        }
    }