Question

我已经实现了三层递归来生成网址的种子列表，然后从每个网址抓取信息。我想使用multiprocessing来利用我系统的所有核心来加速抓取。这是我到目前为止实现的爬虫代码。

# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import HtmlXPathSelector
from scrapy.http.request import Request

from CompanyInfoGrabber.Utility.utils import getAddress, getCompanyStatus, getDirectorDetail, getRegNumber


class CompanyInfoGrabberSpider(scrapy.Spider):
    name = 'CompanyDetail'
    allowed_domains = ['example.com']
    start_urls = ['http://example.com']

    def parse(self, response):
        counter = 0
        print("User Agent in parse() is : ", response.request.headers['User-Agent'])
        hxp = HtmlXPathSelector(response)
        URL_LIST = hxp.select('//sitemapindex/sitemap/loc/text()').extract()
        print("URL LIST: ", URL_LIST)
        for URL in URL_LIST[:2]:
           next_page = response.urljoin(URL)
           yield Request(next_page, self.parse_page)

    def parse_page(self, response):
        print("User Agent in parse_page is : ", response.request.headers['User-Agent'])
        hxp = HtmlXPathSelector(response)

        # create seed list of company-url
        COMPANY_URL_LIST = hxp.select('//urlset/url/loc/text()').extract()
        print("Company url: ", COMPANY_URL_LIST[:20])
        """
        Here I want to use multiprocessing like this
        pool = Pool(processes=8)
        pool.map(parse_company_detail, COMPANY_URL_LIST)
        """
        for company_url in COMPANY_URL_LIST[:5]:
            next_page = response.urljoin(company_url)
            yield Request(next_page, self.parse_company_detail)

    def parse_company_detail(self, response):
        COMPANY_DATA = dict()
        print("User Agent in parse_company_page() is : ", response.request.headers['User-Agent'])
        hxp = HtmlXPathSelector(response)
        _ABOUT_ = ''.join(hxp.xpath('normalize-space(//div[@class="panel-body"]/text())').extract())
        for node in hxp.xpath('//div[@class="panel-body"]//p'):
            _ABOUT_ += ''.join(node.xpath('string()').extract())

        COMPANY_DATA['About'] = _ABOUT_
        # Get company data.
    COMPANY_DATA = getDirectorDetail(COMPANY_DATA, hxp)

    print("Dictionary: ", COMPANY_DATA)
    return COMPANY_DATA

如何使用多处理来抓取网址的种子列表？在此先感谢。

更新我的问题不是this的重复。我这里只有一只蜘蛛。

此致

Om Prakash

Answer 1

我建议使用线程模块同时运行多个线程。您需要修改您的类以在 init 中获取URL参数。

import threading

sites = ['URL1','URL2','URL3']

def create_instance():
    global sites
    CompanyInfoGrabberSpider(scrapy.Spider,sites[0])
    sites.remove[sites[0]]

for site in sites:
    threading.Thread(target=create_instance).start() # Create and start thread

如何在Python中使用多处理来使用Scrapy抓取数百万个URL？

1 个答案: