Question

我知道有一些关键概念，我可以正确地修改代码或指向正确的方向。生病的谷歌，尝试一些事情，但我不知道为什么它不起作用

任何帮助表示赞赏/都会+1

如果您需要更多详细信息，请发表评论进口沙皮导入时间

start_url = 'https://bbb.hidden-street.net/search_finder/shoes%20for%20jump'


class MySpider(scrapy.Spider):
    name = 'MySpider'

    start_urls = [start_url]

    def parse(self, response):
        # print('url:', response.url)
        products = response.xpath('//div/table/tbody')

        for product in products:
            item = {
                #'name': product.xpath('./tr/td/b[1]/a/text()').extract(),
                'link': product.xpath('./tr[3]/div/a/@href').extract(),
            }

            url = response.urlparse.urljoin('https://bbb.hidden-street.net', product[1:])
            yield scrapy.Request(url=url, callback=self.parse_product, meta={'item': item})

    #  yield response.follow(item['link'], callback=self.parse_product, meta={'item': item})

        #time.sleep(5)

        # execute with low
        yield scrapy.Request(start_url, dont_filter=True, priority=-1)

    def parse_product(self, response):
        # print('url:', response.url)

        name = response.xpath('(//strong)[1]/text()').re(r'(\w+)')
        level = response.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "image", " " ))]').re(r'Level:(\s*\d+)')

        hp = response.xpath('//td').re(r'HP:(\s*\d+)')

        #scrolls = response.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "image", " " ))] | //strong+//a//img/@title').re(r'\bScroll\b')

        for price, hp, scrolls in zip(name, hp, level):
            yield {'name': name.strip(), 'hp': hp.strip(), 'scroll':level.strip()}


# --- it runs without project and saves in `output.csv` ---

from scrapy.crawler import CrawlerRunner

def _run_crawler(spider_cls, settings):
    """
    spider_cls: Scrapy Spider class
    returns: Twisted Deferred
    """
    runner = CrawlerRunner(settings)
    return runner.crawl(spider_cls)     # return Deferred


def test_scrapy_crawler():
    deferred = _run_crawler(MySpider, settings)

    @deferred.addCallback
    def _success(results):
        """
        After crawler completes, this function will execute.
        Do your assertions in this function.
        """

    @deferred.addErrback
    def _error(failure):
        raise failure.value

    return deferred

请链接一些资源（更多细节）

Answer 1

删除此行

scrapy.Request（start_url，dont_filter = True，priority = -1）

因为提取数据后，parse（）方法一次又一次在start_url上产生了一个新请求。

webcrawler仅循环遍历starturl

1 个答案: