Question

我已经制作了一只蜘蛛来使用scrapy从这个here页面获取评论。我想要产品评论直到某个日期（在这种情况下，2016年7月2日）。我想在审核日期早于给定日期时关闭我的蜘蛛并返回项目列表。蜘蛛运作良好，但我的问题是，如果满足条件，我无法关闭我的蜘蛛..如果我提出异常，蜘蛛关闭而不返回任何东西。请建议手动关闭蜘蛛的最佳方法。这是我的代码：

from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy import Selector
from tars.items import FlipkartProductReviewsItem
import re as r
import unicodedata
from datetime import datetime 

class Freviewspider(CrawlSpider):
    name = "frs"
    allowed_domains = ["flipkart.com"]
    def __init__(self, *args, **kwargs):
        super(Freviewspider, self).__init__(*args, **kwargs)
        self.start_urls = [kwargs.get('start_url')]


    rules = (
        Rule(LinkExtractor(allow=(), restrict_xpaths=('//a[@class="nav_bar_next_prev"]')), callback="parse_start_url", follow= True),
)


    def parse_start_url(self, response):

        hxs = Selector(response)
        titles = hxs.xpath('//div[@class="fclear fk-review fk-position-relative line "]')

        items = []

        for i in titles:

            item = FlipkartProductReviewsItem()

            #x-paths:

            title_xpath = "div[2]/div[1]/strong/text()"
            review_xpath = "div[2]/p/span/text()"
            date_xpath = "div[1]/div[3]/text()"



            #field-values-extraction:

            item["date"] = (''.join(i.xpath(date_xpath).extract())).replace('\n ', '')
            item["title"] = (''.join(i.xpath(title_xpath).extract())).replace('\n ', '')

            review_list = i.xpath(review_xpath).extract()
            temp_list = []
            for element in review_list:
                temp_list.append(element.replace('\n ', '').replace('\n', ''))

            item["review"] = ' '.join(temp_list)

            xxx = datetime.strptime(item["date"], '%d %b %Y ')
            comp_date = datetime.strptime('02 Jul 2016 ', '%d %b %Y ')
            if xxx>comp_date:
                items.append(item)
            else:
                break

        return(items)

Answer 1

要强制关闭蜘蛛，您可以使用here in scrapy docs所述的引发CloseSpider例外。在提出异常之前，请确保退货/退货。

满足条件时关闭scrapy spider并返回输出对象

1 个答案: