protected function getCrawlRequests() : Generator
{
$i = 0;
while ($crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($i)) {
if (!$this->crawlProfile->shouldCrawl($crawlUrl->url)) {
$i++;
continue;
}
if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) {
$i++;
continue;
}
$this->crawlObserver->willCrawl($crawlUrl->url);
$this->crawlQueue->markAsProcessed($crawlUrl);
(yield new Request('GET', (string) $crawlUrl->url));
$i++;
}
}