让php搜寻器跳过特定的网址

时间:2020-05-06 13:17:58

标签: php laravel web-crawler sitemap

我有一个GenerateSitemap.php文件,可以在其中配置搜寻器,但是我不明白如何使搜寻器跳过某些特定的URL,例如(https://example.com/noindex-url)。我已经读过这篇文章,但是我无法理解。 https://github.com/spatie/laravel-sitemap

namespace Example\Console\Commands;

use Illuminate\Console\Command;
use Spatie\Sitemap\SitemapGenerator;
use Spatie\Sitemap\Tags\Url;
use Spatie\Crawler\Crawler;

class GenerateSitemap extends Command
{
    /**
     * The console command name.
     *
     * @var string
     */
    protected $signature = 'sitemap:generate';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Generate the sitemap.';

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {   
        $siteURL = 'https://example.com';
        SitemapGenerator::create($siteURL)
            ->configureCrawler(function (Crawler $crawler) {
                $crawler->ignoreRobots();
            })
            ->hasCrawled(function (Url $url) {
                if ((string)$url->path() === '/') {
                    return;
                }

                $this->output->writeln('Crawled: ' . (string)$url->path());

                return $url;
            })
            ->writeToFile(public_path('sitemap.xml'));

1 个答案:

答案 0 :(得分:0)

if ((string)$url->path() === '/noindex-url') {
    return;
}

做到了!

这就是现在的样子:

namespace Example\Console\Commands;

use Illuminate\Console\Command;
use Spatie\Sitemap\SitemapGenerator;
use Spatie\Sitemap\Tags\Url;
use Spatie\Crawler\Crawler;

class GenerateSitemap extends Command
{
    /**
     * The console command name.
     *
     * @var string
     */
    protected $signature = 'sitemap:generate';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Generate the sitemap.';

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {   
        $siteURL = 'https://example.com';
        SitemapGenerator::create($siteURL)
            ->configureCrawler(function (Crawler $crawler) {
                $crawler->ignoreRobots();
            })
            ->hasCrawled(function (Url $url) {
                if ((string)$url->path() === '/') {
                    return;
                }

                if ((string)$url->path() === '/noindex-url') {
                    return;
                }
                $this->output->writeln('Crawled: ' . (string)$url->path());

                return $url;
            })
            ->writeToFile(public_path('sitemap.xml'));