Spatie\Crawler\CrawlProfile::shouldCrawl PHP Method

    public function shouldCrawl(Url $url) : bool;

 /**
  * Crawl all links in the given html.
  *
  * @param string $html
  */
 protected function crawlAllLinks($html)
 {
     $allLinks = $this->getAllLinks($html);
     collect($allLinks)->reject(function (Url $url) {
         return $url->isEmailUrl() || $url->isTelUrl() || $url->isJavascript();
     })->map(function (Url $url) {
         return $this->normalizeUrl($url);
     })->filter(function (Url $url) {
         return $this->crawlProfile->shouldCrawl($url);
     })->each(function (Url $url) {
         $this->crawlUrl($url);
     });
 }