public function markAsProcessed(CrawlUrl $crawlUrl)
{
$this->processed->push($crawlUrl);
}
protected function getCrawlRequests() : Generator { $i = 0; while ($crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($i)) { if (!$this->crawlProfile->shouldCrawl($crawlUrl->url)) { $i++; continue; } if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) { $i++; continue; } $this->crawlObserver->willCrawl($crawlUrl->url); $this->crawlQueue->markAsProcessed($crawlUrl); (yield new Request('GET', (string) $crawlUrl->url)); $i++; } }