Question

新手编程/ Python36。操作系统：Windows 10

我在Scrapy中写了一个蜘蛛，我的json文件只返回了一些值。这让我觉得xpath语法是正确的，但我似乎无法找到错误。

This question根本不是一个帮助，因为这不是我的代码段的问题。

代码+输出：

代码

# -*- coding: utf-8 -*-
import scrapy


class OfriSpider(scrapy.Spider):
    name = "ofri"
    allowed_domains = ["www.ofri.ch/firmen"]
    start_urls = ['http://www.ofri.ch/firmen/Abbruchunternehmen/']

    def parse(self, response):
        entries_description = response.xpath('//div[@class="directory-entry"]//div[@class="directory-entry-description"]')
        for entry_description in entries_description:
            company_name = entry_description.xpath('.//h2/a/text()').extract()
            address_street = entry_description.xpath('.//p[@itemprop="address"]/span[@itemprop="streetAddress"]/text()').extract()
            zip_locality = entry_description.xpath('.//p[@itemprop="address"]/span[@itemprop="addressLocality"]/text()').extract()
            contact_data = entry_description.xpath('.//*[@id="business_directory_contact_data"]/div/ul').extract_first()
            tel = entry_description.xpath('.//span[@itemprop="telephone"]//text()').extract()
            company_url = entry_description.xpath('.//a[@itemprop="url"]/@href').extract()
            yield{'name':company_name,
                  'street':address_street,
                  'zip_locality':zip_locality,
                  'tel':tel,
                  'url':company_url}

输出

[
,
{"name": ["Eugen Schnabel Transport & Transfer"], "street": ["Talstrasse 24"], "zip_locality": ["8885 Mols"], "tel": ["0041767198236"], "url": []},
,
,
{"name": ["Hanna-reinigung"], "street": ["Schlierenstrasse 48"], "zip_locality": ["8902 Urdorf"], "tel": ["0799481971"], "url": []},
,
{"name": ["Malergesch\u00e4ft Peic"], "street": ["Affolternstrasse 60"], "zip_locality": ["8105 Regensdorf"], "tel": ["0764824149"], "url": []},
,
,
,
,
{"name": ["Einzelfirma, Michael Heidelberger"], "street": ["Dorfstrasse 151"], "zip_locality": ["8424 Embrach"], "tel": ["0787907234"], "url": []},
{"name": ["ASS Immo + Bau GmbH"], "street": ["Tretteliweg 3b"], "zip_locality": ["8305 Dietlikon"], "tel": ["0764029370"], "url": []},
,
,
{"name": ["Baumontagen R. Schneiter"], "street": ["Jonenbachstrasse 7"], "zip_locality": ["8911 Rifferswil"], "tel": ["0796552188"], "url": []},
{"name": ["Plus Bau & GU GmbH"], "street": ["Wiesentrasse 83"], "zip_locality": ["3014 Bern"], "tel": ["0763462153"], "url": []},
,
,

]

有人发现这里有什么问题吗？

Answer 1

我获取您的代码并创建不使用项目中任何其他文件的独立版本 - 并且获取所有数据没有问题。

如果你创建了项目，可能在设置或其他文件中存在问题。

在某些地方，我将extract()更改为extract_first()，因为总是有单个元素。

#!/usr/bin/env python3

import scrapy

class MySpider(scrapy.Spider):

    name = 'myspider'

    allowed_domains = ['www.ofri.ch/firmen']
    start_urls = ['http://www.ofri.ch/firmen/Abbruchunternehmen/']

    def parse(self, response):
        print('url:', response.url)

        entries_description = response.xpath('//div[@class="directory-entry"]//div[@class="directory-entry-description"]')

        for entry_description in entries_description:
            company_name = entry_description.xpath('.//h2/a/text()').extract_first()
            address_street = entry_description.xpath('.//p[@itemprop="address"]/span[@itemprop="streetAddress"]/text()').extract_first()
            zip_locality = entry_description.xpath('.//p[@itemprop="address"]/span[@itemprop="addressLocality"]/text()').extract_first()
            contact_data = entry_description.xpath('.//*[@id="business_directory_contact_data"]/div/ul').extract_first()
            tel = entry_description.xpath('.//span[@itemprop="telephone"]//text()').extract_first()
            company_url = entry_description.xpath('.//a[@itemprop="url"]/@href').extract_first()

            item = {
                'name': company_name,
                'street': address_street,
                'zip_locality': zip_locality,
                'tel': tel,
                'url':company_url
            }    

            print(item)

            yield item


# --- it runs without project and saves in `output.csv` ---

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',

    # save in file as CSV, JSON or XML
    'FEED_FORMAT': 'json',
    'FEED_URI': 'output.csv',
})
c.crawl(MySpider)
c.start()

结果：

[
{"name": "SZ.AC Team", "street": "Aadorferstrasse 19", "zip_locality": "8362 Balterswil", "tel": "0719601976", "url": "http://allesuntereinemdach.ch"},
{"name": "Eugen Schnabel Transport & Transfer", "street": "Talstrasse 24", "zip_locality": "8885 Mols", "tel": "0041767198236", "url": null},
{"name": "Glarner Rueckbau", "street": "Reimen 1", "zip_locality": "8775 H\u00e4tzingen", "tel": "0794017852", "url": "http://glarner-rueckbau-klg.ch"},
{"name": "Poldeschtrans", "street": "Talgasse 21", "zip_locality": "5503 Schafisheim", "tel": "0764414732", "url": "http://poldesch.ch"},
{"name": "Hanna-reinigung", "street": "Schlierenstrasse 48", "zip_locality": "8902 Urdorf", "tel": "0799481971", "url": null},
{"name": "FREUDENBERG HANDWERK UND INSTALLATIONSSERVICE", "street": "Bahnhofstrasse 2", "zip_locality": "4543 Deitingen", "tel": "0762872322", "url": "http://freudenberg-handwerk.ch"},
{"name": "Malergesch\u00e4ft Peic", "street": "Affolternstrasse 60", "zip_locality": "8105 Regensdorf", "tel": "0764824149", "url": null},
{"name": "Linsin Bau", "street": "M\u00fcllerweg 1", "zip_locality": "4633 Hauenstein", "tel": "0041793390620", "url": "http://Linsin-Bau.ch"},
{"name": "Sanimpex GmbH", "street": "Badenerstrasse 549", "zip_locality": "8048 Z\u00fcrich", "tel": "0432109670", "url": "http://sanimpex.ch"},
{"name": "M.Spangenberg", "street": "Kronenweg.3", "zip_locality": "8165 Oberweningen", "tel": "0434228168", "url": "http://spangenberg-kundenmaurer.ch"},
{"name": "M. Johann Tiefbau", "street": "Bahnhofstrasse 2", "zip_locality": "6162 Entlebuch", "tel": "0415303406", "url": "http://johann-tiefbau.ch"},
{"name": "Einzelfirma, Michael Heidelberger", "street": "Dorfstrasse 151", "zip_locality": "8424 Embrach", "tel": "0787907234", "url": null},
{"name": "ASS Immo + Bau GmbH", "street": "Tretteliweg 3b", "zip_locality": "8305 Dietlikon", "tel": "0764029370", "url": null},
{"name": "tm rent Gmbh", "street": "Im Ebnet 66", "zip_locality": "8700 K\u00fcsnacht", "tel": "0764148714", "url": "http://www.tm-rent.ch"},
{"name": "HBS Bau GmbH", "street": "Eisenbahnstrasse 18", "zip_locality": "8730 Uznach", "tel": "0797343583", "url": "http://hbs-bau.ch"},
{"name": "Baumontagen R. Schneiter", "street": "Jonenbachstrasse 7", "zip_locality": "8911 Rifferswil", "tel": "0796552188", "url": null},
{"name": "Plus Bau & GU GmbH", "street": "Wiesentrasse 83", "zip_locality": "3014 Bern", "tel": "0763462153", "url": null},
{"name": "Ettlin Mont-& Demontagen", "street": "Untere Gr\u00fcndlistrasse 20", "zip_locality": "6055 Alpnach Dorf", "tel": "0794121162", "url": "http://ettlin-montagen.ch"},
{"name": "ISISERVICE", "street": "Tiefenaustrasse 131", "zip_locality": "3004 Bern", "tel": "0313815488", "url": "http://isiservice.ch"},
{"name": "S & G Services GmbH", "street": "Mutschellenstrasse 85", "zip_locality": "8038 Z\u00fcrich", "tel": "0762502222", "url": "http://sg-reinigung.ch"}
]

Answer 2

根据我的检查，您的Xpath是正确的，您可以使用chrome验证主xpath：

$x('//div[@class="directory-entry"]//div[@class="directory-entry-description"]');

所以问题应该在其他地方，可能是不同的div内容。

除了url（它获取对象，而不是url值），例如，查看结果：

$x('//div[@class="directory-entry"]//div[@class="directory-entry-description"]//a[@itemprop="url"]/@href'); 在铬。

Scrapy Spider没有返回所有元素

2 个答案: