Question

我的目标是每页提取所有25行（每行6个项目），然后遍历40页中的每一页。

目前，我的蜘蛛从第1-3页开始提取第一行（参见CSV输出图像）。

我假设list_iterator()函数会迭代每一行;但是，我的rules或list_iterator()函数中似乎存在错误，该错误不允许每页的所有行都被废弃。

非常感谢任何帮助或建议！

propub_spider.py：

import scrapy 
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 
from propub.items import PropubItem
from scrapy.http import Request

class propubSpider(CrawlSpider):
    name = 'prop$'
    allowed_domains = ['https://projects.propublica.org']
    max_pages = 40
    start_urls = [
        'https://projects.propublica.org/docdollars/search?state%5Bid%5D=33',
        'https://projects.propublica.org/docdollars/search?page=2&state%5Bid%5D=33',
        'https://projects.propublica.org/docdollars/search?page=3&state%5Bid%5D=33']

    rules = (Rule(SgmlLinkExtractor(allow=('\\search?page=\\d')), 'parse_start_url', follow=True),)

    def list_iterator(self):
        for i in range(self.max_pages):
            yield Request('https://projects.propublica.org/docdollars/search?page=d' % i, callback=self.parse)

    def parse(self, response):
        for sel in response.xpath('//*[@id="payments_list"]/tbody'):
            item = PropubItem()
            item['payee'] = sel.xpath('tr[1]/td[1]/a[2]/text()').extract()
            item['link'] = sel.xpath('tr[1]/td[1]/a[1]/@href').extract()
            item['city'] = sel.xpath('tr[1]/td[2]/text()').extract()
            item['state'] = sel.xpath('tr[1]/td[3]/text()').extract()
            item['company'] = sel.xpath('tr[1]/td[4]').extract()
            item['amount'] =  sel.xpath('tr[1]/td[7]/span/text()').extract()
            yield item

pipelines.py：

import csv

class PropubPipeline(object):

    def __init__(self):
        self.myCSV = csv.writer(open('C:\Users\Desktop\propub.csv', 'wb'))
        self.myCSV.writerow(['payee', 'link', 'city', 'state', 'company', 'amount'])

    def process_item(self, item, spider):
        self.myCSV.writerow([item['payee'][0].encode('utf-8'), 
        item['link'][0].encode('utf-8'), 
        item['city'][0].encode('utf-8'), 
        item['state'][0].encode('utf-8'),
        item['company'][0].encode('utf-8'),
        item['amount'][0].encode('utf-8')])
        return item

items.py：

import scrapy
from scrapy.item import Item, Field

class PropubItem(scrapy.Item):
    payee = scrapy.Field()
    link = scrapy.Field()
    city = scrapy.Field()
    state = scrapy.Field()
    company = scrapy.Field()
    amount =  scrapy.Field()
    pass

CSV输出：

enter image description here

Answer 1

需要修复多个事项：

使用start_requests()方法代替list_iterator()

此处缺少%：

yield Request('https://projects.propublica.org/docdollars/search?page=%d' % i, callback=self.parse)
#                                                                 HERE^

您不需要CrawlSpider，因为您通过start_requests()提供分页链接 - 使用常规scrapy.Spider
如果XPath表达式与类属性

修正版：

import scrapy

from propub.items import PropubItem


class propubSpider(scrapy.Spider):
    name = 'prop$'
    allowed_domains = ['projects.propublica.org']
    max_pages = 40

    def start_requests(self):
        for i in range(self.max_pages):
            yield scrapy.Request('https://projects.propublica.org/docdollars/search?page=%d' % i, callback=self.parse)

    def parse(self, response):
        for sel in response.xpath('//*[@id="payments_list"]//tr[@data-payment-id]'):
            item = PropubItem()
            item['payee'] = sel.xpath('td[@class="name_and_payee"]/a[last()]/text()').extract()
            item['link'] = sel.xpath('td[@class="name_and_payee"]/a[1]/@href').extract()
            item['city'] = sel.xpath('td[@class="city"]/text()').extract()
            item['state'] = sel.xpath('td[@class="state"]/text()').extract()
            item['company'] = sel.xpath('td[@class="company"]/text()').extract()
            item['amount'] = sel.xpath('td[@class="amount"]/text()').extract()
            yield item

Scrapy：不成功迭代列表和分页

1 个答案: