/**
* @inheritdoc
*/
protected function collectObjects($url)
{
if (!array_key_exists($url, $this->collectedCount)) {
$this->collectedCount[$url] = 0;
}
$host = 'http://' . parse_url($url, PHP_URL_HOST);
$dom = new Dom();
try {
$dom->loadFromUrl($url, [], GlabsController::$curl);
} catch (CurlException $e) {
if (false === strpos($e->getMessage(), 'timed out')) {
throw new CurlException($e->getMessage());
}
GlabsController::showMessage(' ...trying again', false);
return $this->collectObjects($url);
}
if (false !== strpos($dom, 'This IP has been automatically blocked.')) {
throw new CurlException('IP has been blocked.');
}
// end collect. no results
if ($dom->find('#moon')[0]) {
return true;
}
$this->checkTotalObjects($dom);
/* @var \PHPHtmlParser\Dom\AbstractNode $span */
foreach ($dom->find('.txt') as $span) {
if ($this->isEnoughCollect()) {
break;
}
/* @var \PHPHtmlParser\Dom\AbstractNode $link */
if ($link = $span->find('a')[0]) {
$href = $this->checkObjectLink($host, $link->getAttribute('href'));
if (false === $href) {
continue;
}
$title = $link->text() ?: strip_tags($link->innerHtml());
try {
$object = $this->getObjectModel($url, $href, $title, $this->categoryId, $this->type);
$object->setPrice($span);
} catch (ObjectException $e) {
continue;
}
$this->collected[] = $href;
$this->objects[] = $object;
$this->collectedCount[$url]++;
BaseSite::$doneObjects++;
BaseSite::progress();
}
}
if (!$this->isEnoughCollect()) {
$curl = GlabsController::$curl;
$curl::$referer = $url;
$url = str_replace([self::$pageParam . self::$page, '#list'], '', $url);
self::$page += 100;
return $this->collectObjects($this->getPagedUrl($url));
}
return true;
}