Question

我调整了Using Middleware to ignore duplicates in Scrapy的代码。

from scrapy.exceptions import DropItem
from scrapy import log
import os.path

class IgnoreDuplicates():

    def __init__(self):
        self._cu_file = open("crawled_urls.txt", "a+")
        self._crawled_urls = set([line.strip() for line in self._cu_file.readlines()])

    def process_request(self, request, spider):
        if request.url in self._crawled_urls:
            raise DropItem("Duplicate product scrape caught by IgnoreDuplicates at <%s>" % (url))
        else:
            self._crawled_urls.add(request.url)
            self._cu_file.write(request.url + '\n')
            log.msg("IgnoreDuplicates recorded this url " + request.url, level=log.DEBUG)
            return None

我还将中间件模块添加到了settings.py：

SPIDER_MANAGER_CLASS = 'slybot.spidermanager.SlybotSpiderManager'
EXTENSIONS = {'slybot.closespider.SlybotCloseSpider': 1}
ITEM_PIPELINES = {'slybot.dupefilter.DupeFilterPipeline': 1}
SPIDER_MIDDLEWARES = {'slybot.middleware.IgnoreDuplicates': 500, 'slybot.spiderlets.SpiderletsMiddleware': 999}  # as close as possible to spider output
PLUGINS = ['slybot.plugins.scrapely_annotations.Annotations']
SLYDUPEFILTER_ENABLED = True
PROJECT_DIR = 'slybot-project'
FEED_EXPORTERS = {
    'csv': 'slybot.exporter.SlybotCSVItemExporter',
}
CSV_EXPORT_FIELDS = None

try:
    from local_slybot_settings import *
except ImportError:
    pass

不会调用process_request函数。我尝试在settings.py中更改中间件密钥的值，以便在SpiderletsMiddleware之前和之后执行。但是异常和日志消息不会显示在输出中。

如何确保调用中间件？

Answer 1

蜘蛛中间件的回调函数不同。我使用此代码段中的代码作为参考： http://snipplr.com/view/67018/middleware-to-avoid-revisiting-already-visited-items/

这是我在问题中发布的中间件代码的工作版本。

from scrapy.http import Request
from scrapy import log
import os.path

class IgnoreVisitedItems(object):
    def __init__(self):
        # Load the URLs that have already been crawled
        self._cu_file = open("crawled_urls.txt", "a+")
        self._crawled_urls = set([line.strip() for line in self._cu_file.readlines()])

    def process_spider_output(self, response, result, spider):
        ret = []
        for x in result:
            # Find the URL in the result or response
            url = None
            if isinstance(x, Request):
                url = x.url
            else:
                url = response.request.url

            # Check if the URL has been crawled, and add
            # it to the list of crawled URLs.
            if url in self._crawled_urls:
                log.msg("Ignoring already visited: %s" % url,
                        level=log.INFO, spider=spider)
            else:
                log.msg("Adding %s to list of visited urls" % url,
                        level=log.INFO, spider=spider)
                self._cu_file.write(url + '\n')
                self._crawled_urls.add(url)
                ret.append(x)
        return ret

Portia中的Spider中间件未调用

1 个答案: