Question

我要抓取以下网站：https://graphics.stltoday.com/apps/payrolls/salaries/teachers/

希望抓取每个人的所有数据。这意味着要链接到每个地区，然后再链接到该地区内的每个工作类别，最后是每个员工。我认为问题可能出在我的网址正则表达式上，但我不确定。在每个员工的页面上，我认为我已经正确识别了XPath：

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class Spider2(CrawlSpider):
    #name of the spider
    name = 'stltoday'

    #list of allowed domains
    allowed_domains = ['graphics.stltoday.com']

    #starting url for scraping
    start_urls = ['https://graphics.stltoday.com/apps/payrolls/salaries/teachers']

    rules = [
    Rule(LinkExtractor(
        allow=['/[0-9]+/$']),
        follow=True),
    Rule(LinkExtractor(
        allow=['/[0-9]+/position/[0-9]+/$']),
        follow=True),
    Rule(LinkExtractor(
        allow=['/detail/[0-9]+/$']),
        callback='parse_item',
        follow=True),
    ]

    #setting the location of the output csv file
    custom_settings = {
        'FEED_FORMAT' : "csv",
        'FEED_URI' : 'tmp/stltoday1.csv'
    }

    def parse_item(self, response):
        #Remove XML namespaces
        response.selector.remove_namespaces()
        url = response.url
        #Extract article information

        fullname = response.xpath('//p[@class="table__title"]./text()').extract_first()

        for row in response.xpath('//th[@scope="row"]'):
            yield {
            "url": url,
        "fullname": fullname,
            "district": row.xpath('./text()').extract_first(),
            "school": row.xpath('./following-sibling::*[1]/text()').extract_first(),
            "degree": row.xpath('./following-sibling::*[2]/text()').extract_first(),
            "salary": row.xpath('./following-sibling::*[3]/text()').extract_first(),
        "extcontractpay": row.xpath('./following-sibling::*[4]/text()').extract_first(),
        "extraduty": row.xpath('./following-sibling::*[5]/text()').extract_first(),
        "totalpay": row.xpath('./following-sibling::*[6]/text()').extract_first(),
        "yearsindistrict": row.xpath('./following-sibling::*[7]/text()').extract_first(),
        "yearsinmoschools": row.xpath('./following-sibling::*[8]/text()').extract_first(),
            }


        for item in zip(url,fullname,district,school,degree,salary,extcontractpay,extraduty,totalpay,yearsindistrict,yearsinmoschools):
            yield {
                'url' : url,
        'fullname' : fullname,
                'district' : district,
                'school' : school,
                'degree' : degree,
                'salary' : salary,
        'extcontractpay' : extcontractpay,
                'extraduty' : extraduty,
                'totalpay' : totalpay,
                'yearsindistrict' : yearsindistrict,
                'yearsinmoschools' : yearsinmoschools
            }

蜘蛛运行（我暂停了几分钟后），但没有任何内容写入.csv文件。

Answer 1

因此，我沿着一个兔子洞走了，将蜘蛛重建为一个基本的蜘蛛，而不是爬行。我不明白为什么在LinkEctract规则集中没有回叫解析器。

无论如何，我创建了一个cvs_exporter函数来更好地管理输出。将其及其参数添加到设置和瞧。

蜘蛛通过与“爬行”蜘蛛相同的逻辑穿越站点，尽管将目标指定为网址，而不是广泛的抓取。从 “ parse_district”>“ parse_postions”>最终改为“ parse_person”，其中您要刮擦的物品存在。

#stlSpider.py
import scrapy
from stltoday.items import StltodayItem

class StlspiderSpider(scrapy.Spider):
    name = 'stlSpider'
    allowed_domains = ['graphics.stltoday.com']
    start_urls = ['http://graphics.stltoday.com/apps/payrolls/salaries/teachers/']

    def parse(self, response):
        for href in response.xpath("//th/a/@href").re(".*/teachers/[0-9]+/"):
            yield scrapy.Request(response.urljoin(href),
                                 callback=self.parse_district)

    def parse_district(self, response):
        for href in response.xpath("//th/a/@href").re(".*position.*"):
            yield scrapy.Request(response.urljoin(href),
                                 callback=self.parse_position)

    def parse_position(self, response):
        for href in response.xpath("//td/a/@href").extract():
            yield scrapy.Request(response.urljoin(href),
                                 callback=self.parse_person)

    def parse_person(self, response):
        item = StltodayItem()
        name = response.xpath('//p[@class="table__title"]/text()').extract_first()
        row = response.xpath('//th[@scope="row"]')
        item["url"] = response.url
        item["fullname"] = name
        item["district"] = row.xpath('//th[contains(., "District")]/following-sibling::td/text()').extract_first()
        item["school"] = row.xpath('//th[contains(., "School")]/following-sibling::td/text()').extract_first()
        item["degree"] = row.xpath('//th[contains(., "Degree")]/following-sibling::td/text()').extract_first()
        item["salary"] = row.xpath('//th[contains(., "Salary")]/following-sibling::td/text()').extract_first()
        item["extcontractpay"] = row.xpath('//th[contains(., "Extended")]/following-sibling::td/text()').extract_first()
        item["extraduty"] = row.xpath('//th[contains(., "Extra")]/following-sibling::td/text()').extract_first()
        item["totalpay"] = row.xpath('//th[contains(., "Total")]/following-sibling::td/text()').extract_first()
        item["yearsindistrict"] = row.xpath('//th[contains(., "Years in district")]/following-sibling::td/text()').extract_first()
        item["yearsinmoschools"] = row.xpath('//th[contains(., "Years in MO")]/following-sibling::td/text()').extract_first()
        yield item

逐项列出...项目大声笑

#items.py
import scrapy


class StltodayItem(scrapy.Item):
    url = scrapy.Field()
    fullname = scrapy.Field()
    district = scrapy.Field()
    school = scrapy.Field()
    degree = scrapy.Field()
    salary = scrapy.Field()
    extcontractpay = scrapy.Field()
    extraduty = scrapy.Field()
    totalpay = scrapy.Field()
    yearsindistrict = scrapy.Field()
    yearsinmoschools = scrapy.Field()

创建了一个“ csv_exporter”模块，您可以在其中调用该模块调整文件输出的方式，包括设置定界符和要输出项目的顺序

#csv_exporter.py
_author_ = 'Erick'
from scrapy.conf import settings
from scrapy.contrib.exporter import CsvItemExporter

class MyProjectCsvItemExporter(CsvItemExporter):

    def __init__(self, *args, **kwargs):
        delimiter = settings.get('CSV_DELIMITER', ',')
        kwargs['delimiter'] = delimiter

        fields_to_export = settings.get('FIELDS_TO_EXPORT', [])
        if fields_to_export :
            kwargs['fields_to_export'] = fields_to_export

        super(MyProjectCsvItemExporter, self).__init__(*args, **kwargs)

将导出程序包括到您的settings.py文件中，此处包括 args将ins“ csv_exporter”设置为您要使用的分隔符，并且出口项目的顺序

#settings.py
OT_NAME = 'stltoday'

SPIDER_MODULES = ['stltoday.spiders']
NEWSPIDER_MODULE = 'stltoday.spiders'
FEED_FORMAT = 'csv'
FEED_URI = 'tmp/stltoday1.csv'
FIELDS_TO_EXPORT = ["url", "fullname", "district", "school", "degree", "salary", "extcontractpay", "extraduty", "totalpay", "yearsindistrict", "yearsinmoschools"]
FEED_EXPORTERS = {
    'csv': 'stltoday.csv_exporter.MyProjectCsvItemExporter',
}
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'stltoday (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
...

未写入草率结果

1 个答案: