protected function startCrawlingQueue()
{
while ($this->crawlQueue->hasPendingUrls()) {
$pool = new Pool($this->client, $this->getCrawlRequests(), ['concurrency' => $this->concurrency, 'options' => [RequestOptions::CONNECT_TIMEOUT => 10, RequestOptions::TIMEOUT => 10, RequestOptions::ALLOW_REDIRECTS => false], 'fulfilled' => function (ResponseInterface $response, int $index) {
$this->handleResponse($response, $index);
$crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($index);
if ($crawlUrl->url->host !== $this->baseUrl->host) {
return;
}
$this->addAllLinksToCrawlQueue((string) $response->getBody(), $crawlUrl = $this->crawlQueue->getPendingUrlAtIndex($index)->url);
}, 'rejected' => function (RequestException $exception, int $index) {
$this->handleResponse($exception->getResponse(), $index);
}]);
$promise = $pool->promise();
$promise->wait();
$this->crawlQueue->removeProcessedUrlsFromPending();
}
}